diff --git a/docker/rocm/migraphx/CMakeLists.txt b/docker/rocm/migraphx/CMakeLists.txt index 271dd094b..29a258df0 100644 --- a/docker/rocm/migraphx/CMakeLists.txt +++ b/docker/rocm/migraphx/CMakeLists.txt @@ -13,7 +13,7 @@ SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) project(migraphx_py) -include_directories(/opt/rocm/include) +include_directories(/opt/rocm/include /opt/rocm/lib) find_package(pybind11 REQUIRED) pybind11_add_module(migraphx migraphx_py.cpp) diff --git a/docker/rocm/migraphx/adjust_allocation.cpp b/docker/rocm/migraphx/adjust_allocation.cpp deleted file mode 100644 index 3dad3b7fa..000000000 --- a/docker/rocm/migraphx/adjust_allocation.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void adjust_allocation::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - // skip instruction with no input - if(ins->inputs().empty()) - continue; - - // Skip target-independent operators - if(ins->get_operator().is_context_free()) - continue; - - auto alias_ins = instruction::get_output_alias(ins, true); - if(alias_ins->name() != model.name() and alias_ins->name() != "@param") - continue; - // shape allocated is different from actual shape - // of the instruction, reallocate and replace the previous one - if(alias_ins->get_shape() == ins->get_shape()) - continue; - auto alloc_ins = m.insert_instruction(ins, model.allocate(ins->get_shape())); - m.replace_instruction(alias_ins, alloc_ins); - // If the memory is an output parameter then copy the memory to the parameter - if(alias_ins->name() == "@param") - { - auto copy = m.insert_instruction(std::next(ins), make_op(model.copy()), ins, alias_ins); - auto tail = range(std::next(copy), m.end()); - for(auto i : iterator_for(tail)) - { - if(contains(i->inputs(), ins)) - instruction::replace_argument(i, ins, copy); - } - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/analyze_streams.cpp b/docker/rocm/migraphx/analyze_streams.cpp deleted file mode 100644 index 0f8ac68f8..000000000 --- a/docker/rocm/migraphx/analyze_streams.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -bool happens_before(const std::vector& e1, const std::vector& e2) -{ - return std::equal(e1.begin(), e1.end(), e2.begin(), e2.end(), std::less_equal<>{}) and - not std::equal(e1.begin(), e1.end(), e2.begin(), e2.end(), std::greater_equal<>{}); -} - -std::vector analyze_streams(const module& m, const stream_model& strmm) -{ - using vector_clock = std::vector; - std::vector races; - auto nstream = strmm.get_nstream(); - std::vector vclock(nstream, vector_clock(nstream)); - std::unordered_map timestamp; - std::unordered_map events; - for(auto ins : iterator_for(m)) - { - if(not strmm.has_stream(ins)) - continue; - std::size_t s = strmm.get_stream(ins); - assert(s < nstream); - assert(vclock.size() == nstream); - assert(vclock[s].size() == nstream); - if(strmm.is_record(ins)) - { - vclock[s][s]++; - auto event = strmm.get_event_id(ins); - events[event] = vclock[s]; - } - else if(strmm.is_wait(ins)) - { - auto event = strmm.get_event_id(ins); - if(not contains(events, event)) - MIGRAPHX_THROW("Event is waited on before being recorded: " + - std::to_string(event)); - auto payload = events.at(event); - assert(vclock[s].size() == payload.size()); - std::transform(vclock[s].begin(), - vclock[s].end(), - payload.begin(), - vclock[s].begin(), - [&](auto x, auto y) { return std::max(x, y); }); - vclock[s][s]++; - } - else - { - vclock[s][s]++; - } - timestamp[ins] = vclock[s]; - } - for(auto ins : iterator_for(m)) - { - if(not strmm.has_stream(ins)) - continue; - if(ins->inputs().empty()) - continue; - std::size_t s = strmm.get_stream(ins); - // Find inputs from different streams - std::vector inputs; - fix([&](auto self, auto start) { - for(auto input : start->inputs()) - { - if(not strmm.has_stream(input)) - self(input); - else if(strmm.get_stream(input) != s) - inputs.push_back(input); - } - })(ins); - auto it = std::find_if(inputs.begin(), inputs.end(), [&](auto input) { - return not happens_before(timestamp.at(input), timestamp.at(ins)); - }); - if(it != inputs.end()) - { - races.push_back({ins, *it}); - } - } - - return races; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/api/CMakeLists.txt b/docker/rocm/migraphx/api/CMakeLists.txt deleted file mode 100644 index 9c6139738..000000000 --- a/docker/rocm/migraphx/api/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -add_library(migraphx_c - api.cpp -) -set_target_properties(migraphx_c PROPERTIES EXPORT_NAME c) -migraphx_generate_export_header(migraphx_c DIRECTORY migraphx/api) - -# migraphx_c is stable API interface library. SO version of this should be -# bumped when binary compatibility is broken. -rocm_set_soversion(migraphx_c 3.0) - -if(BUILD_TESTING) - target_compile_definitions(migraphx_c PRIVATE MIGRAPHX_BUILD_TESTING) -endif() - -rocm_clang_tidy_check(migraphx_c) -target_link_libraries(migraphx_c PRIVATE migraphx migraphx_tf migraphx_onnx) - -rocm_install_targets( - TARGETS migraphx_c - INCLUDE - ${CMAKE_CURRENT_SOURCE_DIR}/include -) diff --git a/docker/rocm/migraphx/api/api.cpp b/docker/rocm/migraphx/api/api.cpp deleted file mode 100644 index 4c4c1d8f2..000000000 --- a/docker/rocm/migraphx/api/api.cpp +++ /dev/null @@ -1,2442 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -#ifdef MIGRAPHX_BUILD_TESTING -static thread_local bool disable_exception_catch = false; // NOLINT - -extern "C" MIGRAPHX_C_EXPORT void migraphx_test_private_disable_exception_catch(bool b) -{ - disable_exception_catch = b; -} -#endif - -template -migraphx_status try_(F f, bool output = true) // NOLINT -{ -#ifdef MIGRAPHX_BUILD_TESTING - if(disable_exception_catch) - { - f(); - } - else - { -#endif - try - { - f(); - } - catch(const migraphx::exception& ex) - { - if(output) - std::cerr << "MIGraphX Error: " << ex.what() << std::endl; - if(ex.error > 0) - return migraphx_status(ex.error); - else - return migraphx_status_unknown_error; - } - catch(const std::exception& ex) - { - if(output) - std::cerr << "MIGraphX Error: " << ex.what() << std::endl; - return migraphx_status_unknown_error; - } - catch(...) - { - return migraphx_status_unknown_error; - } -#ifdef MIGRAPHX_BUILD_TESTING - } -#endif - return migraphx_status_success; -} - -shape::type_t to_shape_type(migraphx_shape_datatype_t t) -{ - switch(t) - { - case migraphx_shape_tuple_type: return shape::tuple_type; -#define MIGRAPHX_DETAIL_SHAPE_CASE_CONVERT(x, y) \ - case migraphx_shape_##x: return shape::x; - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_DETAIL_SHAPE_CASE_CONVERT) -#undef MIGRAPHX_DETAIL_SHAPE_CASE_CONVERT - } - MIGRAPHX_THROW(migraphx_status_bad_param, "Unknown type"); -} - -migraphx_shape_datatype_t to_shape_type(shape::type_t t) -{ - switch(t) - { - case shape::tuple_type: return migraphx_shape_tuple_type; -#define MIGRAPHX_DETAIL_SHAPE_CASE_CONVERT(x, y) \ - case shape::x: return migraphx_shape_##x; - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_DETAIL_SHAPE_CASE_CONVERT) -#undef MIGRAPHX_DETAIL_SHAPE_CASE_CONVERT - } - MIGRAPHX_THROW(migraphx_status_bad_param, "Unknown type"); -} - -template -auto to_obj_vector(const T* x, std::size_t n) -{ - std::vectorobject)> result; - std::transform(x, x + n, std::back_inserter(result), [&](auto&& y) { return y->object; }); - return result; -} - -template -auto to_objptr_vector(const U* x, std::size_t n) -{ - std::vector result; - std::transform( - x, x + n, std::back_inserter(result), [&](auto&& y) { return std::addressof(y->object); }); - return result; -} - -target get_target(const std::string& name) { return make_target(name); } - -void set_offload_copy(compile_options& options, bool value) { options.offload_copy = value; } - -void set_fast_math(compile_options& options, bool value) { options.fast_math = value; } - -void set_exhaustive_tune_flag(compile_options& options, bool value) -{ - options.exhaustive_tune = value; -} - -void set_file_format(file_options& options, const char* format) { options.format = format; } - -void set_default_dim_value(onnx_options& options, size_t value) -{ - options.default_dim_value = value; -} - -void set_default_dyn_dim_value(onnx_options& options, const shape::dynamic_dimension& dd) -{ - options.default_dyn_dim_value = dd; -} - -void set_default_loop_iterations(onnx_options& options, int64_t value) -{ - options.max_loop_iterations = value; -} - -void set_external_data_path(onnx_options& options, const char* external_data_path) -{ - options.external_data_path = std::string(external_data_path); -} - -void set_limit_loop_iterations(onnx_options& options, int64_t value) -{ - options.limit_max_iterations = value; -} - -void set_nhwc(tf_options& options, bool is_nhwc) { options.is_nhwc = is_nhwc; } - -void set_default_dim_value(tf_options& options, size_t value) { options.batch_size = value; } - -void set_input_parameter_shape(onnx_options& options, - const char* name, - std::vector dims) -{ - options.map_input_dims[std::string(name)] = std::move(dims); -} - -void set_dyn_input_parameter_shape(onnx_options& options, - const char* name, - std::vector dyn_dims) -{ - options.map_dyn_input_dims[std::string(name)] = std::move(dyn_dims); -} - -void set_input_parameter_shape(tf_options& options, const char* name, std::vector dims) -{ - options.map_input_dims[std::string(name)] = std::move(dims); -} - -void set_output_names(tf_options& options, std::vector names) -{ - options.output_node_names = std::vector(names.begin(), names.end()); -} - -std::vector -run_async(program& p, const parameter_map& params, void* s, std::string_view name) -{ - execution_environment exec_env{any_ptr(s, name), true}; - return p.eval(params, exec_env); -} - -template -std::vector get_names(const std::unordered_map& m) -{ - std::vector result; - std::transform( - m.begin(), m.end(), std::back_inserter(result), [](auto&& p) { return p.first.c_str(); }); - return result; -} - -template -std::set make_set(const T* x, std::size_t n) -{ - return {x, x + n}; -} - -void quantize_fp16_with_op_names(program& prog, std::vector& names) -{ - if(names.empty()) - { - names = {"all"}; - } - - migraphx::quantize_fp16(prog, names); -} - -struct quantize_int8_options -{ - std::vector calibration = {}; - std::unordered_set op_names = {}; -}; - -void add_op_name(quantize_int8_options& options, const char* name) -{ - options.op_names.insert(name); -} - -void add_calibration_data(quantize_int8_options& options, parameter_map& data) -{ - options.calibration.push_back(data); -} - -void quantize_int8_wrap(program& prog, const target& t, quantize_int8_options& options) -{ - if(options.op_names.empty()) - { - options.op_names = {"dot", "convolution"}; - } - - migraphx::quantize_int8(prog, t, options.calibration, options.op_names); -} - -struct quantize_fp8_options -{ - std::vector calibration = {}; -}; - -void add_calibration_data(quantize_fp8_options& options, parameter_map& data) -{ - options.calibration.push_back(data); -} - -void quantize_fp8_wrap(program& prog, const target& t, quantize_fp8_options& options) -{ - migraphx::quantize_fp8(prog, t, options.calibration); -} - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wformat-nonliteral" -#endif - -operation create_op(const char* name, const char* attributes, va_list vlist) -{ - std::string sattributes = attributes == nullptr ? "" : attributes; - std::vector buffer(sattributes.size() * 2); - std::vsnprintf(buffer.data(), buffer.size(), sattributes.c_str(), vlist); - value v = value::object{}; - if(attributes != nullptr) - { - v = from_json_string(convert_to_json(std::string(buffer.data()))); - } - auto op = make_op(name, v); - - return op; -} - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -template -bool equal(const T& x, const T& y) -{ - return x == y; -} - -std::vector run(program& p, const parameter_map& params) { return p.eval(params); } - -std::vector get_output_shapes(program& p) { return p.get_output_shapes(); } - -void print_program(const program& p) { std::cout << p << std::endl; } - -void print_module(const module& m) { std::cout << m << std::endl; } - -migraphx::instruction_ref add_allocation(module& m, const migraphx::shape& s) -{ - return m.add_instruction(migraphx::make_op("allocate", {{"shape", migraphx::to_value(s)}}), {}); -} - -struct experimental_custom_op -{ - std::string name; - experimental_custom_op() = default; - - experimental_custom_op(std::string pname) : name(std::move(pname)) {} -}; - -template -struct custom_operation -{ - - template - static auto reflect(Self&, F) - { - return pack(); - } - - value attributes() const - { - return {{"custom_op", true}, {"target", op.runs_on_offload_target() ? "gpu" : "cpu"}}; - } - - CustomOp op; - std::string name() const { return op.xobject.name; } - - shape compute_shape(std::vector inputs) const - { - return op.compute_shape(std::move(inputs)); - } - - // TODO: Compute method with module_args - argument - compute(migraphx::context ctx, migraphx::shape output_shape, std::vector inputs) const - { - return op.compute(std::move(ctx), std::move(output_shape), std::move(inputs)); - } - - std::ptrdiff_t output_alias(std::vector inputs) const - { - auto alias_vec = op.output_alias(std::move(inputs)); - // TODO: For now, only support one output alias - if(alias_vec.empty()) - { - return -1; - } - if(alias_vec.size() > 1) - { - MIGRAPHX_THROW("Currently, CustomOps in MIGraphX only supports one output_alias"); - } - return alias_vec.front(); - } - - bool runs_on_offload_target() const { return op.runs_on_offload_target(); } -}; - -template -void register_custom_op(const CustomOp& op) -{ - register_op(custom_operation{op}); -} - -migraphx::context get_context(const program& p) { return p.get_context(); } - -} // namespace migraphx - -template > -Target* object_cast(U* x) -{ - return reinterpret_cast(x); -} -template > -const Target* object_cast(const U* x) -{ - return reinterpret_cast(x); -} - -template > -Target* allocate(Ts&&... xs) -{ - if constexpr(std::is_aggregate{}) - return new Target{std::forward(xs)...}; // NOLINT - else - return new Target(std::forward(xs)...); // NOLINT -} - -template -void destroy(T* x) -{ - delete x; // NOLINT -} - -// TODO: Move to interface preamble -template -struct manage_generic_ptr -{ - manage_generic_ptr() = default; - - manage_generic_ptr(std::nullptr_t) {} - - manage_generic_ptr(void* pdata, const char* obj_tname, C pcopier, D pdeleter) - : data(nullptr), obj_typename(obj_tname), copier(pcopier), deleter(pdeleter) - { - copier(&data, pdata); - } - - manage_generic_ptr(const manage_generic_ptr& rhs) - : data(nullptr), obj_typename(rhs.obj_typename), copier(rhs.copier), deleter(rhs.deleter) - { - if(copier) - copier(&data, rhs.data); - } - - manage_generic_ptr(manage_generic_ptr&& other) noexcept - : data(other.data), - obj_typename(other.obj_typename), - copier(other.copier), - deleter(other.deleter) - { - other.data = nullptr; - other.obj_typename = ""; - other.copier = nullptr; - other.deleter = nullptr; - } - - manage_generic_ptr& operator=(manage_generic_ptr rhs) - { - std::swap(data, rhs.data); - std::swap(obj_typename, rhs.obj_typename); - std::swap(copier, rhs.copier); - std::swap(deleter, rhs.deleter); - return *this; - } - - ~manage_generic_ptr() - { - if(data != nullptr) - deleter(data); - } - - void* data = nullptr; - const char* obj_typename = ""; - C copier = nullptr; - D deleter = nullptr; -}; - -extern "C" struct migraphx_optimals; -struct migraphx_optimals -{ - template - migraphx_optimals(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::set object; -}; - -extern "C" struct migraphx_dynamic_dimension; -struct migraphx_dynamic_dimension -{ - template - migraphx_dynamic_dimension(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::shape::dynamic_dimension object; -}; - -extern "C" struct migraphx_dynamic_dimensions; -struct migraphx_dynamic_dimensions -{ - template - migraphx_dynamic_dimensions(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::vector object; -}; - -extern "C" struct migraphx_shape; -struct migraphx_shape -{ - template - migraphx_shape(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::shape object; -}; - -extern "C" struct migraphx_argument; -struct migraphx_argument -{ - template - migraphx_argument(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::argument object; -}; - -extern "C" struct migraphx_target; -struct migraphx_target -{ - template - migraphx_target(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::target object; -}; - -extern "C" struct migraphx_program_parameter_shapes; -struct migraphx_program_parameter_shapes -{ - template - migraphx_program_parameter_shapes(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::unordered_map object; -}; - -extern "C" struct migraphx_program_parameters; -struct migraphx_program_parameters -{ - template - migraphx_program_parameters(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::unordered_map object; -}; - -extern "C" struct migraphx_arguments; -struct migraphx_arguments -{ - template - migraphx_arguments(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::vector object; -}; - -extern "C" struct migraphx_shapes; -struct migraphx_shapes -{ - template - migraphx_shapes(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::vector object; -}; - -extern "C" struct migraphx_instruction; -struct migraphx_instruction -{ - template - migraphx_instruction(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::instruction_ref object; -}; - -extern "C" struct migraphx_instructions; -struct migraphx_instructions -{ - template - migraphx_instructions(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::vector object; -}; - -extern "C" struct migraphx_modules; -struct migraphx_modules -{ - template - migraphx_modules(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::vector object; -}; - -extern "C" struct migraphx_module; -struct migraphx_module -{ - template - migraphx_module(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::module object; -}; - -extern "C" struct migraphx_program; -struct migraphx_program -{ - template - migraphx_program(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::program object; -}; - -extern "C" struct migraphx_operation; -struct migraphx_operation -{ - template - migraphx_operation(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::operation object; -}; - -extern "C" struct migraphx_onnx_options; -struct migraphx_onnx_options -{ - template - migraphx_onnx_options(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::onnx_options object; -}; - -extern "C" struct migraphx_file_options; -struct migraphx_file_options -{ - template - migraphx_file_options(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::file_options object; -}; - -extern "C" struct migraphx_compile_options; -struct migraphx_compile_options -{ - template - migraphx_compile_options(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::compile_options object; -}; - -extern "C" struct migraphx_tf_options; -struct migraphx_tf_options -{ - template - migraphx_tf_options(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::tf_options object; -}; - -extern "C" struct migraphx_quantize_op_names; -struct migraphx_quantize_op_names -{ - template - migraphx_quantize_op_names(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - std::vector object; -}; - -extern "C" struct migraphx_quantize_int8_options; -struct migraphx_quantize_int8_options -{ - template - migraphx_quantize_int8_options(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::quantize_int8_options object; -}; - -extern "C" struct migraphx_quantize_fp8_options; -struct migraphx_quantize_fp8_options -{ - template - migraphx_quantize_fp8_options(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::quantize_fp8_options object; -}; - -extern "C" struct migraphx_context; -struct migraphx_context -{ - template - migraphx_context(Ts&&... xs) - : object(std::forward(xs)...) // NOLINT(readability-redundant-member-init) - { - } - migraphx::context object; -}; - -extern "C" struct migraphx_experimental_custom_op; -struct migraphx_experimental_custom_op -{ - template - migraphx_experimental_custom_op(void* p, - migraphx_experimental_custom_op_copy c, - migraphx_experimental_custom_op_delete d, - const char* obj_typename, - Ts&&... xs) - : object_ptr(p, obj_typename, c, d), xobject(std::forward(xs)...) - { - } - manage_generic_ptr - object_ptr = nullptr; - migraphx::experimental_custom_op xobject; - migraphx_experimental_custom_op_compute compute_f = nullptr; - migraphx::argument compute(migraphx::context ctx, - migraphx::shape output, - std::vector inputs) const - { - if(compute_f == nullptr) - throw std::runtime_error("compute function is missing."); - std::remove_pointer_t out; - std::array exception_msg; - exception_msg.front() = '\0'; - auto api_error_result = compute_f(&out, - object_ptr.data, - exception_msg.data(), - exception_msg.size(), - object_cast(&(ctx)), - object_cast(&(output)), - object_cast(&(inputs))); - if(api_error_result != migraphx_status_success) - { - const std::string exception_str(exception_msg.data()); - throw std::runtime_error("Error in compute of: " + - std::string(object_ptr.obj_typename) + ": " + exception_str); - } - return (&out)->object; - } - - migraphx_experimental_custom_op_compute_shape compute_shape_f = nullptr; - migraphx::shape compute_shape(std::vector inputs) const - { - if(compute_shape_f == nullptr) - throw std::runtime_error("compute_shape function is missing."); - std::remove_pointer_t out; - std::array exception_msg; - exception_msg.front() = '\0'; - auto api_error_result = compute_shape_f(&out, - object_ptr.data, - exception_msg.data(), - exception_msg.size(), - object_cast(&(inputs))); - if(api_error_result != migraphx_status_success) - { - const std::string exception_str(exception_msg.data()); - throw std::runtime_error("Error in compute_shape of: " + - std::string(object_ptr.obj_typename) + ": " + exception_str); - } - return (&out)->object; - } - - migraphx_experimental_custom_op_output_alias output_alias_f = nullptr; - std::vector output_alias(std::vector inputs) const - { - if(output_alias_f == nullptr) - throw std::runtime_error("output_alias function is missing."); - std::array out; - std::remove_pointer_t out_size = 1024; - std::array exception_msg; - exception_msg.front() = '\0'; - auto api_error_result = output_alias_f(out.data(), - &out_size, - object_ptr.data, - exception_msg.data(), - exception_msg.size(), - object_cast(&(inputs))); - if(api_error_result != migraphx_status_success) - { - const std::string exception_str(exception_msg.data()); - throw std::runtime_error("Error in output_alias of: " + - std::string(object_ptr.obj_typename) + ": " + exception_str); - } - return {out.begin(), out.begin() + out_size}; // cppcheck-suppress returnDanglingLifetime; - } - - migraphx_experimental_custom_op_runs_on_offload_target runs_on_offload_target_f = nullptr; - bool runs_on_offload_target() const - { - if(runs_on_offload_target_f == nullptr) - throw std::runtime_error("runs_on_offload_target function is missing."); - std::remove_pointer_t out; - std::array exception_msg; - exception_msg.front() = '\0'; - auto api_error_result = runs_on_offload_target_f( - &out, object_ptr.data, exception_msg.data(), exception_msg.size()); - if(api_error_result != migraphx_status_success) - { - const std::string exception_str(exception_msg.data()); - throw std::runtime_error("Error in runs_on_offload_target of: " + - std::string(object_ptr.obj_typename) + ": " + exception_str); - } - return out; - } -}; - -extern "C" migraphx_status migraphx_optimals_destroy(migraphx_optimals_t optimals) -{ - auto api_error_result = migraphx::try_([&] { destroy((optimals)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_optimals_assign_to(migraphx_optimals_t output, - const_migraphx_optimals_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_optimals_create(migraphx_optimals_t* optimals, const size_t* ptr, size_t size) -{ - auto api_error_result = migraphx::try_([&] { - *optimals = object_cast( - allocate>(migraphx::make_set((ptr), (size)))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimension_destroy(migraphx_dynamic_dimension_t dynamic_dimension) -{ - auto api_error_result = migraphx::try_([&] { destroy((dynamic_dimension)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimension_assign_to(migraphx_dynamic_dimension_t output, - const_migraphx_dynamic_dimension_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_dynamic_dimension_create_min_max( - migraphx_dynamic_dimension_t* dynamic_dimension, size_t min, size_t max) -{ - auto api_error_result = migraphx::try_([&] { - *dynamic_dimension = object_cast( - allocate((min), (max))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimension_create_min_max_optimals(migraphx_dynamic_dimension_t* dynamic_dimension, - size_t min, - size_t max, - migraphx_optimals_t optimals) -{ - auto api_error_result = migraphx::try_([&] { - if(optimals == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter optimals: Null pointer"); - *dynamic_dimension = object_cast( - allocate((min), (max), (optimals->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimension_is_fixed(bool* out, const_migraphx_dynamic_dimension_t dynamic_dimension) -{ - auto api_error_result = migraphx::try_([&] { - if(dynamic_dimension == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter dynamic_dimension: Null pointer"); - *out = (dynamic_dimension->object).is_fixed(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimension_equal(bool* out, - const_migraphx_dynamic_dimension_t dynamic_dimension, - const_migraphx_dynamic_dimension_t x) -{ - auto api_error_result = migraphx::try_([&] { - if(dynamic_dimension == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter dynamic_dimension: Null pointer"); - if(x == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter x: Null pointer"); - *out = migraphx::equal((dynamic_dimension->object), (x->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimensions_destroy(migraphx_dynamic_dimensions_t dynamic_dimensions) -{ - auto api_error_result = migraphx::try_([&] { destroy((dynamic_dimensions)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimensions_assign_to(migraphx_dynamic_dimensions_t output, - const_migraphx_dynamic_dimensions_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimensions_create(migraphx_dynamic_dimensions_t* dynamic_dimensions, - const const_migraphx_dynamic_dimension_t* ptr, - size_t size) -{ - auto api_error_result = migraphx::try_([&] { - *dynamic_dimensions = object_cast( - allocate>( - migraphx::to_obj_vector((ptr), (size)))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimensions_size(size_t* out, migraphx_dynamic_dimensions_t dynamic_dimensions) -{ - auto api_error_result = migraphx::try_([&] { - if(dynamic_dimensions == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter dynamic_dimensions: Null pointer"); - *out = (dynamic_dimensions->object).size(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_dynamic_dimensions_get(const_migraphx_dynamic_dimension_t* out, - migraphx_dynamic_dimensions_t dynamic_dimensions, - size_t idx) -{ - auto api_error_result = migraphx::try_([&] { - if(dynamic_dimensions == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter dynamic_dimensions: Null pointer"); - *out = object_cast( - &((dynamic_dimensions->object).at((idx)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_destroy(migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { destroy((shape)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_assign_to(migraphx_shape_t output, - const_migraphx_shape_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_create(migraphx_shape_t* shape, - migraphx_shape_datatype_t type, - size_t* lengths, - size_t lengths_size) -{ - auto api_error_result = migraphx::try_([&] { - if(lengths == nullptr and lengths_size != 0) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter lengths: Null pointer"); - *shape = object_cast( - allocate((migraphx::to_shape_type(type)), - (std::vector(lengths, lengths + lengths_size)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_create_with_strides(migraphx_shape_t* shape, - migraphx_shape_datatype_t type, - size_t* lengths, - size_t lengths_size, - size_t* strides, - size_t strides_size) -{ - auto api_error_result = migraphx::try_([&] { - if(lengths == nullptr and lengths_size != 0) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter lengths: Null pointer"); - if(strides == nullptr and strides_size != 0) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter strides: Null pointer"); - *shape = object_cast( - allocate((migraphx::to_shape_type(type)), - (std::vector(lengths, lengths + lengths_size)), - (std::vector(strides, strides + strides_size)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_create_scalar(migraphx_shape_t* shape, - migraphx_shape_datatype_t type) -{ - auto api_error_result = migraphx::try_([&] { - *shape = object_cast( - allocate((migraphx::to_shape_type(type)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_create_dynamic(migraphx_shape_t* shape, - migraphx_shape_datatype_t type, - migraphx_dynamic_dimensions_t dims) -{ - auto api_error_result = migraphx::try_([&] { - if(dims == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dims: Null pointer"); - *shape = object_cast( - allocate((migraphx::to_shape_type(type)), (dims->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_shape_lengths(const size_t** out, size_t* out_size, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(out == nullptr or out_size == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter out: Null pointer"); - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - auto&& api_result = (shape->object).lens(); - *out = api_result.data(); - *out_size = api_result.size(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_shape_strides(const size_t** out, size_t* out_size, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(out == nullptr or out_size == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter out: Null pointer"); - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - auto&& api_result = (shape->object).strides(); - *out = api_result.data(); - *out_size = api_result.size(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_dyn_dims(migraphx_dynamic_dimensions_t* out, - const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = allocate((shape->object).dyn_dims()); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_type(migraphx_shape_datatype_t* out, - const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(out == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter out: Null pointer"); - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = migraphx::to_shape_type((shape->object).type()); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_elements(size_t* out, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = (shape->object).elements(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_bytes(size_t* out, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = (shape->object).bytes(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_ndim(size_t* out, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = (shape->object).ndim(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_shape_equal(bool* out, const_migraphx_shape_t shape, const_migraphx_shape_t x) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - if(x == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter x: Null pointer"); - *out = migraphx::equal((shape->object), (x->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_standard(bool* out, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = (shape->object).standard(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_dynamic(bool* out, const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = (shape->object).dynamic(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shape_index(size_t* out, const_migraphx_shape_t shape, size_t i) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = (shape->object).index((i)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_argument_destroy(migraphx_argument_t argument) -{ - auto api_error_result = migraphx::try_([&] { destroy((argument)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_argument_assign_to(migraphx_argument_t output, - const_migraphx_argument_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_argument_create(migraphx_argument_t* argument, const_migraphx_shape_t shape, void* buffer) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *argument = object_cast( - allocate((shape->object), (buffer))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_argument_create_empty(migraphx_argument_t* argument, - const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *argument = object_cast(allocate((shape->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_argument_shape(const_migraphx_shape_t* out, - const_migraphx_argument_t argument) -{ - auto api_error_result = migraphx::try_([&] { - if(argument == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter argument: Null pointer"); - *out = object_cast(&((argument->object).get_shape())); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_argument_buffer(char** out, const_migraphx_argument_t argument) -{ - auto api_error_result = migraphx::try_([&] { - if(argument == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter argument: Null pointer"); - *out = (argument->object).data(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_argument_equal(bool* out, const_migraphx_argument_t argument, const_migraphx_argument_t x) -{ - auto api_error_result = migraphx::try_([&] { - if(argument == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter argument: Null pointer"); - if(x == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter x: Null pointer"); - *out = migraphx::equal((argument->object), (x->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_argument_generate(migraphx_argument_t* out, const_migraphx_shape_t s, size_t seed) -{ - auto api_error_result = migraphx::try_([&] { - if(s == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter s: Null pointer"); - *out = allocate(migraphx::generate_argument((s->object), (seed))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_target_destroy(migraphx_target_t target) -{ - auto api_error_result = migraphx::try_([&] { destroy((target)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_target_assign_to(migraphx_target_t output, - const_migraphx_target_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_target_create(migraphx_target_t* target, const char* name) -{ - auto api_error_result = migraphx::try_([&] { - *target = object_cast( - allocate(migraphx::get_target((name)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_parameter_shapes_destroy( - migraphx_program_parameter_shapes_t program_parameter_shapes) -{ - auto api_error_result = migraphx::try_([&] { destroy((program_parameter_shapes)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameter_shapes_assign_to(migraphx_program_parameter_shapes_t output, - const_migraphx_program_parameter_shapes_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameter_shapes_size(size_t* out, - migraphx_program_parameter_shapes_t program_parameter_shapes) -{ - auto api_error_result = migraphx::try_([&] { - if(program_parameter_shapes == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter program_parameter_shapes: Null pointer"); - *out = (program_parameter_shapes->object).size(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameter_shapes_get(const_migraphx_shape_t* out, - migraphx_program_parameter_shapes_t program_parameter_shapes, - const char* name) -{ - auto api_error_result = migraphx::try_([&] { - if(program_parameter_shapes == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter program_parameter_shapes: Null pointer"); - *out = - object_cast(&((program_parameter_shapes->object).at((name)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_parameter_shapes_names( - const char** out, migraphx_program_parameter_shapes_t program_parameter_shapes) -{ - auto api_error_result = migraphx::try_([&] { - if(out == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter out: Null pointer"); - if(program_parameter_shapes == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter program_parameter_shapes: Null pointer"); - auto&& api_result = migraphx::get_names((program_parameter_shapes->object)); - std::copy(api_result.begin(), api_result.end(), out); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameters_destroy(migraphx_program_parameters_t program_parameters) -{ - auto api_error_result = migraphx::try_([&] { destroy((program_parameters)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameters_assign_to(migraphx_program_parameters_t output, - const_migraphx_program_parameters_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameters_create(migraphx_program_parameters_t* program_parameters) -{ - auto api_error_result = migraphx::try_([&] { - *program_parameters = object_cast( - allocate>()); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_parameters_add(migraphx_program_parameters_t program_parameters, - const char* name, - const_migraphx_argument_t argument) -{ - auto api_error_result = migraphx::try_([&] { - if(program_parameters == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter program_parameters: Null pointer"); - if(argument == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter argument: Null pointer"); - (program_parameters->object)[(name)] = (argument->object); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_arguments_destroy(migraphx_arguments_t arguments) -{ - auto api_error_result = migraphx::try_([&] { destroy((arguments)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_arguments_assign_to(migraphx_arguments_t output, - const_migraphx_arguments_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_arguments_size(size_t* out, migraphx_arguments_t arguments) -{ - auto api_error_result = migraphx::try_([&] { - if(arguments == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter arguments: Null pointer"); - *out = (arguments->object).size(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_arguments_get(const_migraphx_argument_t* out, migraphx_arguments_t arguments, size_t idx) -{ - auto api_error_result = migraphx::try_([&] { - if(arguments == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter arguments: Null pointer"); - *out = object_cast(&((arguments->object).at((idx)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shapes_destroy(migraphx_shapes_t shapes) -{ - auto api_error_result = migraphx::try_([&] { destroy((shapes)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shapes_assign_to(migraphx_shapes_t output, - const_migraphx_shapes_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_shapes_size(size_t* out, migraphx_shapes_t shapes) -{ - auto api_error_result = migraphx::try_([&] { - if(shapes == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shapes: Null pointer"); - *out = (shapes->object).size(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_shapes_get(const_migraphx_shape_t* out, migraphx_shapes_t shapes, size_t idx) -{ - auto api_error_result = migraphx::try_([&] { - if(shapes == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shapes: Null pointer"); - *out = object_cast(&((shapes->object).at((idx)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_instruction_destroy(migraphx_instruction_t instruction) -{ - auto api_error_result = migraphx::try_([&] { destroy((instruction)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_instruction_assign_to(migraphx_instruction_t output, - const_migraphx_instruction_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_instructions_destroy(migraphx_instructions_t instructions) -{ - auto api_error_result = migraphx::try_([&] { destroy((instructions)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_instructions_assign_to(migraphx_instructions_t output, - const_migraphx_instructions_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_instructions_create(migraphx_instructions_t* instructions, - const const_migraphx_instruction_t* ptr, - size_t size) -{ - auto api_error_result = migraphx::try_([&] { - *instructions = - object_cast(allocate>( - migraphx::to_obj_vector((ptr), (size)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_modules_destroy(migraphx_modules_t modules) -{ - auto api_error_result = migraphx::try_([&] { destroy((modules)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_modules_assign_to(migraphx_modules_t output, - const_migraphx_modules_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_modules_create(migraphx_modules_t* modules, migraphx_module_t* ptr, size_t size) -{ - auto api_error_result = migraphx::try_([&] { - *modules = object_cast(allocate>( - migraphx::to_objptr_vector((ptr), (size)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_create(migraphx_module_t* module, char* name) -{ - auto api_error_result = migraphx::try_([&] { - if(name == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter name: Null pointer"); - *module = object_cast(allocate((std::string(name)))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_print(const_migraphx_module_t module) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - migraphx::print_module((module->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_add_instruction(migraphx_instruction_t* out, - migraphx_module_t module, - migraphx_operation_t op, - migraphx_instructions_t args) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - if(op == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter op: Null pointer"); - if(args == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter args: Null pointer"); - *out = allocate( - (module->object).add_instruction((op->object), (args->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_module_add_instruction_with_mod_args(migraphx_instruction_t* out, - migraphx_module_t module, - migraphx_operation_t op, - migraphx_instructions_t args, - migraphx_modules_t module_refs) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - if(op == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter op: Null pointer"); - if(args == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter args: Null pointer"); - if(module_refs == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module_refs: Null pointer"); - *out = allocate( - (module->object).add_instruction((op->object), (args->object), (module_refs->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_add_literal(migraphx_instruction_t* out, - migraphx_module_t module, - const_migraphx_shape_t shape, - const char* buffer) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = allocate( - (module->object).add_literal((shape->object), (buffer))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_add_parameter(migraphx_instruction_t* out, - migraphx_module_t module, - const char* name, - const_migraphx_shape_t shape) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - if(shape == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter shape: Null pointer"); - *out = allocate( - (module->object).add_parameter((name), (shape->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_add_return(migraphx_instruction_t* out, - migraphx_module_t module, - migraphx_instructions_t args) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - if(args == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter args: Null pointer"); - *out = allocate((module->object).add_return((args->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_module_add_allocation(migraphx_instruction_t* out, - migraphx_module_t module, - const_migraphx_shape_t s) -{ - auto api_error_result = migraphx::try_([&] { - if(module == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter module: Null pointer"); - if(s == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter s: Null pointer"); - *out = allocate( - migraphx::add_allocation((module->object), (s->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_destroy(migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { destroy((program)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_assign_to(migraphx_program_t output, - const_migraphx_program_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_create(migraphx_program_t* program) -{ - auto api_error_result = migraphx::try_( - [&] { *program = object_cast(allocate()); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_get_main_module(migraphx_module_t* out, - migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - *out = object_cast((program->object).get_main_module()); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_create_module(migraphx_module_t* out, migraphx_program_t program, const char* name) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - *out = object_cast((program->object).create_module((name))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_compile(migraphx_program_t program, - migraphx_target_t target, - migraphx_compile_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - if(target == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter target: Null pointer"); - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - (program->object).compile((target->object), (options->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_get_parameter_shapes(migraphx_program_parameter_shapes_t* out, - migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - *out = - allocate((program->object).get_parameter_shapes()); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_get_output_shapes(migraphx_shapes_t* out, - migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - *out = allocate(migraphx::get_output_shapes((program->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_print(const_migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - migraphx::print_program((program->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_sort(migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - (program->object).sort(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_run(migraphx_arguments_t* out, - migraphx_program_t program, - migraphx_program_parameters_t params) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - if(params == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter params: Null pointer"); - *out = allocate(migraphx::run((program->object), (params->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_program_run_async(migraphx_arguments_t* out, - migraphx_program_t program, - migraphx_program_parameters_t params, - void* s, - const char* name) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - if(params == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter params: Null pointer"); - *out = allocate( - migraphx::run_async((program->object), (params->object), (s), (name))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_equal(bool* out, const_migraphx_program_t program, const_migraphx_program_t x) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - if(x == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter x: Null pointer"); - *out = migraphx::equal((program->object), (x->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_program_experimental_get_context(migraphx_context_t* out, const_migraphx_program_t program) -{ - auto api_error_result = migraphx::try_([&] { - if(program == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter program: Null pointer"); - *out = allocate(migraphx::get_context((program->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_operation_destroy(migraphx_operation_t operation) -{ - auto api_error_result = migraphx::try_([&] { destroy((operation)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_operation_assign_to(migraphx_operation_t output, - const_migraphx_operation_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_operation_create(migraphx_operation_t* operation, - const char* name, - const char* attributes, - ...) -{ - va_list vlist; - va_start(vlist, attributes); - auto api_error_result = migraphx::try_([&] { - *operation = object_cast( - allocate(migraphx::create_op((name), (attributes), (vlist)))); - }); - va_end(vlist); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_operation_name(char* out, size_t out_size, migraphx_operation_t operation) -{ - auto api_error_result = migraphx::try_([&] { - if(out == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter out: Null pointer"); - if(operation == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter operation: Null pointer"); - auto&& api_result = (operation->object).name(); - auto* it = std::copy_n(api_result.begin(), std::min(api_result.size(), out_size - 1), out); - *it = '\0'; - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_load(migraphx_program_t* out, const char* name, migraphx_file_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - *out = allocate(migraphx::load((name), (options->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_save(migraphx_program_t p, const char* name, migraphx_file_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(p == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter p: Null pointer"); - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - migraphx::save((p->object), (name), (options->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_onnx_options_destroy(migraphx_onnx_options_t onnx_options) -{ - auto api_error_result = migraphx::try_([&] { destroy((onnx_options)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_onnx_options_assign_to(migraphx_onnx_options_t output, - const_migraphx_onnx_options_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_onnx_options_create(migraphx_onnx_options_t* onnx_options) -{ - auto api_error_result = migraphx::try_([&] { - *onnx_options = object_cast(allocate()); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_onnx_options_set_input_parameter_shape( - migraphx_onnx_options_t onnx_options, const char* name, size_t* dims, size_t dims_size) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - if(dims == nullptr and dims_size != 0) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dims: Null pointer"); - migraphx::set_input_parameter_shape( - (onnx_options->object), (name), (std::vector(dims, dims + dims_size))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_onnx_options_set_dyn_input_parameter_shape( - migraphx_onnx_options_t onnx_options, const char* name, migraphx_dynamic_dimensions_t dims) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - if(dims == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dims: Null pointer"); - migraphx::set_dyn_input_parameter_shape((onnx_options->object), (name), (dims->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_onnx_options_set_default_dim_value(migraphx_onnx_options_t onnx_options, size_t value) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - migraphx::set_default_dim_value((onnx_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_onnx_options_set_default_dyn_dim_value(migraphx_onnx_options_t onnx_options, - const_migraphx_dynamic_dimension_t dd) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - if(dd == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dd: Null pointer"); - migraphx::set_default_dyn_dim_value((onnx_options->object), (dd->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_onnx_options_set_default_loop_iterations(migraphx_onnx_options_t onnx_options, - int64_t value) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - migraphx::set_default_loop_iterations((onnx_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_onnx_options_set_limit_loop_iterations(migraphx_onnx_options_t onnx_options, int64_t value) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - migraphx::set_limit_loop_iterations((onnx_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_onnx_options_set_external_data_path(migraphx_onnx_options_t onnx_options, - const char* external_data_path) -{ - auto api_error_result = migraphx::try_([&] { - if(onnx_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter onnx_options: Null pointer"); - migraphx::set_external_data_path((onnx_options->object), (external_data_path)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_file_options_destroy(migraphx_file_options_t file_options) -{ - auto api_error_result = migraphx::try_([&] { destroy((file_options)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_file_options_assign_to(migraphx_file_options_t output, - const_migraphx_file_options_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_file_options_create(migraphx_file_options_t* file_options) -{ - auto api_error_result = migraphx::try_([&] { - *file_options = object_cast(allocate()); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_file_options_set_file_format(migraphx_file_options_t file_options, const char* format) -{ - auto api_error_result = migraphx::try_([&] { - if(file_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter file_options: Null pointer"); - migraphx::set_file_format((file_options->object), (format)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_compile_options_destroy(migraphx_compile_options_t compile_options) -{ - auto api_error_result = migraphx::try_([&] { destroy((compile_options)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_compile_options_assign_to(migraphx_compile_options_t output, - const_migraphx_compile_options_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_compile_options_create(migraphx_compile_options_t* compile_options) -{ - auto api_error_result = migraphx::try_([&] { - *compile_options = - object_cast(allocate()); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_compile_options_set_offload_copy(migraphx_compile_options_t compile_options, bool value) -{ - auto api_error_result = migraphx::try_([&] { - if(compile_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter compile_options: Null pointer"); - migraphx::set_offload_copy((compile_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_compile_options_set_fast_math(migraphx_compile_options_t compile_options, bool value) -{ - auto api_error_result = migraphx::try_([&] { - if(compile_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter compile_options: Null pointer"); - migraphx::set_fast_math((compile_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_compile_options_set_exhaustive_tune_flag(migraphx_compile_options_t compile_options, - bool value) -{ - auto api_error_result = migraphx::try_([&] { - if(compile_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter compile_options: Null pointer"); - migraphx::set_exhaustive_tune_flag((compile_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_parse_onnx(migraphx_program_t* out, const char* name, migraphx_onnx_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - *out = allocate(migraphx::parse_onnx((name), (options->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_parse_onnx_buffer(migraphx_program_t* out, - const void* data, - size_t size, - migraphx_onnx_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - *out = allocate( - migraphx::parse_onnx_buffer((data), (size), (options->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_tf_options_destroy(migraphx_tf_options_t tf_options) -{ - auto api_error_result = migraphx::try_([&] { destroy((tf_options)); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_tf_options_assign_to(migraphx_tf_options_t output, - const_migraphx_tf_options_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_tf_options_create(migraphx_tf_options_t* tf_options) -{ - auto api_error_result = migraphx::try_([&] { - *tf_options = object_cast(allocate()); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_tf_options_set_nhwc(migraphx_tf_options_t tf_options, - bool is_nhwc) -{ - auto api_error_result = migraphx::try_([&] { - if(tf_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter tf_options: Null pointer"); - migraphx::set_nhwc((tf_options->object), (is_nhwc)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_tf_options_set_input_parameter_shape( - migraphx_tf_options_t tf_options, const char* name, size_t* dims, size_t dims_size) -{ - auto api_error_result = migraphx::try_([&] { - if(tf_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter tf_options: Null pointer"); - if(dims == nullptr and dims_size != 0) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter dims: Null pointer"); - migraphx::set_input_parameter_shape( - (tf_options->object), (name), (std::vector(dims, dims + dims_size))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_tf_options_set_default_dim_value(migraphx_tf_options_t tf_options, size_t value) -{ - auto api_error_result = migraphx::try_([&] { - if(tf_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter tf_options: Null pointer"); - migraphx::set_default_dim_value((tf_options->object), (value)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_tf_options_set_output_names(migraphx_tf_options_t tf_options, - const char** names, - size_t names_size) -{ - auto api_error_result = migraphx::try_([&] { - if(tf_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter tf_options: Null pointer"); - if(names == nullptr and names_size != 0) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter names: Null pointer"); - migraphx::set_output_names((tf_options->object), - (std::vector(names, names + names_size))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_parse_tf(migraphx_program_t* out, const char* name, migraphx_tf_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - *out = allocate(migraphx::parse_tf((name), (options->object))); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_op_names_destroy(migraphx_quantize_op_names_t quantize_op_names) -{ - auto api_error_result = migraphx::try_([&] { destroy((quantize_op_names)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_op_names_assign_to(migraphx_quantize_op_names_t output, - const_migraphx_quantize_op_names_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_op_names_create(migraphx_quantize_op_names_t* quantize_op_names) -{ - auto api_error_result = migraphx::try_([&] { - *quantize_op_names = - object_cast(allocate>()); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_op_names_add(migraphx_quantize_op_names_t quantize_op_names, const char* name) -{ - auto api_error_result = migraphx::try_([&] { - if(quantize_op_names == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter quantize_op_names: Null pointer"); - (quantize_op_names->object).push_back((name)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_quantize_fp16_with_op_names(migraphx_program_t prog, - migraphx_quantize_op_names_t name) -{ - auto api_error_result = migraphx::try_([&] { - if(prog == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter prog: Null pointer"); - if(name == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter name: Null pointer"); - migraphx::quantize_fp16_with_op_names((prog->object), (name->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_quantize_fp16(migraphx_program_t prog) -{ - auto api_error_result = migraphx::try_([&] { - if(prog == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter prog: Null pointer"); - migraphx::quantize_fp16((prog->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_int8_options_destroy(migraphx_quantize_int8_options_t quantize_int8_options) -{ - auto api_error_result = migraphx::try_([&] { destroy((quantize_int8_options)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_int8_options_assign_to(migraphx_quantize_int8_options_t output, - const_migraphx_quantize_int8_options_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_int8_options_create(migraphx_quantize_int8_options_t* quantize_int8_options) -{ - auto api_error_result = migraphx::try_([&] { - *quantize_int8_options = object_cast( - allocate()); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_int8_options_add_op_name(migraphx_quantize_int8_options_t quantize_int8_options, - const char* name) -{ - auto api_error_result = migraphx::try_([&] { - if(quantize_int8_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter quantize_int8_options: Null pointer"); - migraphx::add_op_name((quantize_int8_options->object), (name)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_quantize_int8_options_add_calibration_data( - migraphx_quantize_int8_options_t quantize_int8_options, migraphx_program_parameters_t data) -{ - auto api_error_result = migraphx::try_([&] { - if(quantize_int8_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter quantize_int8_options: Null pointer"); - if(data == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter data: Null pointer"); - migraphx::add_calibration_data((quantize_int8_options->object), (data->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_quantize_int8(migraphx_program_t prog, - migraphx_target_t target, - migraphx_quantize_int8_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(prog == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter prog: Null pointer"); - if(target == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter target: Null pointer"); - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - migraphx::quantize_int8_wrap((prog->object), (target->object), (options->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_fp8_options_destroy(migraphx_quantize_fp8_options_t quantize_fp8_options) -{ - auto api_error_result = migraphx::try_([&] { destroy((quantize_fp8_options)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_fp8_options_assign_to(migraphx_quantize_fp8_options_t output, - const_migraphx_quantize_fp8_options_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_quantize_fp8_options_create(migraphx_quantize_fp8_options_t* quantize_fp8_options) -{ - auto api_error_result = migraphx::try_([&] { - *quantize_fp8_options = object_cast( - allocate()); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_quantize_fp8_options_add_calibration_data( - migraphx_quantize_fp8_options_t quantize_fp8_options, migraphx_program_parameters_t data) -{ - auto api_error_result = migraphx::try_([&] { - if(quantize_fp8_options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter quantize_fp8_options: Null pointer"); - if(data == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter data: Null pointer"); - migraphx::add_calibration_data((quantize_fp8_options->object), (data->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_quantize_fp8(migraphx_program_t prog, - migraphx_target_t target, - migraphx_quantize_fp8_options_t options) -{ - auto api_error_result = migraphx::try_([&] { - if(prog == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter prog: Null pointer"); - if(target == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter target: Null pointer"); - if(options == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter options: Null pointer"); - migraphx::quantize_fp8_wrap((prog->object), (target->object), (options->object)); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_context_finish(const_migraphx_context_t context) -{ - auto api_error_result = migraphx::try_([&] { - if(context == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter context: Null pointer"); - (context->object).finish(); - }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_context_get_queue(void** out, migraphx_context_t context) -{ - auto api_error_result = migraphx::try_([&] { - if(context == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, "Bad parameter context: Null pointer"); - *out = (context->object).get_queue().unsafe_get(); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_experimental_custom_op_destroy(migraphx_experimental_custom_op_t experimental_custom_op) -{ - auto api_error_result = migraphx::try_([&] { destroy((experimental_custom_op)); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_experimental_custom_op_assign_to(migraphx_experimental_custom_op_t output, - const_migraphx_experimental_custom_op_t input) -{ - auto api_error_result = migraphx::try_([&] { *output = *input; }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_experimental_custom_op_create(migraphx_experimental_custom_op_t* experimental_custom_op, - void* obj, - migraphx_experimental_custom_op_copy c, - migraphx_experimental_custom_op_delete d, - const char* obj_typename, - const char* name) -{ - auto api_error_result = migraphx::try_([&] { - *experimental_custom_op = - allocate((obj), (c), (d), (obj_typename), (name)); - }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_experimental_custom_op_set_compute(migraphx_experimental_custom_op_t obj, - migraphx_experimental_custom_op_compute input) -{ - auto api_error_result = migraphx::try_([&] { (obj)->compute_f = (input); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_experimental_custom_op_set_compute_shape( - migraphx_experimental_custom_op_t obj, migraphx_experimental_custom_op_compute_shape input) -{ - auto api_error_result = migraphx::try_([&] { (obj)->compute_shape_f = (input); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_experimental_custom_op_set_output_alias(migraphx_experimental_custom_op_t obj, - migraphx_experimental_custom_op_output_alias input) -{ - auto api_error_result = migraphx::try_([&] { (obj)->output_alias_f = (input); }); - return api_error_result; -} - -extern "C" migraphx_status migraphx_experimental_custom_op_set_runs_on_offload_target( - migraphx_experimental_custom_op_t obj, - migraphx_experimental_custom_op_runs_on_offload_target input) -{ - auto api_error_result = migraphx::try_([&] { (obj)->runs_on_offload_target_f = (input); }); - return api_error_result; -} - -extern "C" migraphx_status -migraphx_experimental_custom_op_register(migraphx_experimental_custom_op_t experimental_custom_op) -{ - auto api_error_result = migraphx::try_([&] { - if(experimental_custom_op == nullptr) - MIGRAPHX_THROW(migraphx_status_bad_param, - "Bad parameter experimental_custom_op: Null pointer"); - migraphx::register_custom_op((*experimental_custom_op)); - }); - return api_error_result; -} diff --git a/docker/rocm/migraphx/api/include/migraphx/migraphx.h b/docker/rocm/migraphx/api/include/migraphx/migraphx.h deleted file mode 100644 index 155d7926a..000000000 --- a/docker/rocm/migraphx/api/include/migraphx/migraphx.h +++ /dev/null @@ -1,684 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_C_API_MIGRAPHX_H -#define MIGRAPHX_GUARD_C_API_MIGRAPHX_H - -#include -#include -#include - -#include - -// Add new types here -// clang-format off -#define MIGRAPHX_SHAPE_VISIT_TYPES(m) \ - m(bool_type, bool) \ - m(half_type, half) \ - m(float_type, float) \ - m(double_type, double) \ - m(uint8_type, uint8_t) \ - m(int8_type, int8_t) \ - m(uint16_type, uint16_t) \ - m(int16_type, int16_t) \ - m(int32_type, int32_t) \ - m(int64_type, int64_t) \ - m(uint32_type, uint32_t) \ - m(uint64_type, uint64_t) \ - m(fp8e4m3fnuz_type, migraphx::fp8::fp8e4m3fnuz) \ - m(fp8e4m3fn_type, migraphx::fp8::fp8e4m3fn) \ - m(fp8e5m2_type, migraphx::fp8::fp8e5m2) \ - m(bf16_type, bf16) \ - m(fp8e5m2fnuz_type, migraphx::fp8::fp8e5m2fnuz) -// clang-format on - -#ifdef __cplusplus -extern "C" { -#endif - -// return code, more to be added later -typedef enum -{ - migraphx_status_success = 0, - migraphx_status_bad_param = 1, - migraphx_status_unknown_target = 3, - migraphx_status_unknown_error = 4, - -} migraphx_status; - -#define MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES(x, t) migraphx_shape_##x, -/// An enum to represent the different data type inputs -typedef enum -{ - migraphx_shape_tuple_type, - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES) -} migraphx_shape_datatype_t; -#undef MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES - -typedef struct migraphx_optimals* migraphx_optimals_t; -typedef const struct migraphx_optimals* const_migraphx_optimals_t; - -typedef struct migraphx_dynamic_dimension* migraphx_dynamic_dimension_t; -typedef const struct migraphx_dynamic_dimension* const_migraphx_dynamic_dimension_t; - -typedef struct migraphx_dynamic_dimensions* migraphx_dynamic_dimensions_t; -typedef const struct migraphx_dynamic_dimensions* const_migraphx_dynamic_dimensions_t; - -typedef struct migraphx_shape* migraphx_shape_t; -typedef const struct migraphx_shape* const_migraphx_shape_t; - -typedef struct migraphx_argument* migraphx_argument_t; -typedef const struct migraphx_argument* const_migraphx_argument_t; - -typedef struct migraphx_target* migraphx_target_t; -typedef const struct migraphx_target* const_migraphx_target_t; - -typedef struct migraphx_program_parameter_shapes* migraphx_program_parameter_shapes_t; -typedef const struct migraphx_program_parameter_shapes* const_migraphx_program_parameter_shapes_t; - -typedef struct migraphx_program_parameters* migraphx_program_parameters_t; -typedef const struct migraphx_program_parameters* const_migraphx_program_parameters_t; - -typedef struct migraphx_arguments* migraphx_arguments_t; -typedef const struct migraphx_arguments* const_migraphx_arguments_t; - -typedef struct migraphx_shapes* migraphx_shapes_t; -typedef const struct migraphx_shapes* const_migraphx_shapes_t; - -typedef struct migraphx_instruction* migraphx_instruction_t; -typedef const struct migraphx_instruction* const_migraphx_instruction_t; - -typedef struct migraphx_instructions* migraphx_instructions_t; -typedef const struct migraphx_instructions* const_migraphx_instructions_t; - -typedef struct migraphx_modules* migraphx_modules_t; -typedef const struct migraphx_modules* const_migraphx_modules_t; - -typedef struct migraphx_module* migraphx_module_t; -typedef const struct migraphx_module* const_migraphx_module_t; - -typedef struct migraphx_program* migraphx_program_t; -typedef const struct migraphx_program* const_migraphx_program_t; - -typedef struct migraphx_operation* migraphx_operation_t; -typedef const struct migraphx_operation* const_migraphx_operation_t; - -typedef struct migraphx_onnx_options* migraphx_onnx_options_t; -typedef const struct migraphx_onnx_options* const_migraphx_onnx_options_t; - -typedef struct migraphx_file_options* migraphx_file_options_t; -typedef const struct migraphx_file_options* const_migraphx_file_options_t; - -typedef struct migraphx_compile_options* migraphx_compile_options_t; -typedef const struct migraphx_compile_options* const_migraphx_compile_options_t; - -typedef struct migraphx_tf_options* migraphx_tf_options_t; -typedef const struct migraphx_tf_options* const_migraphx_tf_options_t; - -typedef struct migraphx_quantize_op_names* migraphx_quantize_op_names_t; -typedef const struct migraphx_quantize_op_names* const_migraphx_quantize_op_names_t; - -typedef struct migraphx_quantize_int8_options* migraphx_quantize_int8_options_t; -typedef const struct migraphx_quantize_int8_options* const_migraphx_quantize_int8_options_t; - -typedef struct migraphx_quantize_fp8_options* migraphx_quantize_fp8_options_t; -typedef const struct migraphx_quantize_fp8_options* const_migraphx_quantize_fp8_options_t; - -typedef struct migraphx_context* migraphx_context_t; -typedef const struct migraphx_context* const_migraphx_context_t; - -typedef struct migraphx_experimental_custom_op* migraphx_experimental_custom_op_t; -typedef const struct migraphx_experimental_custom_op* const_migraphx_experimental_custom_op_t; - -typedef migraphx_status (*migraphx_experimental_custom_op_compute)(migraphx_argument_t out, - void* obj, - char* exception_msg, - size_t exception_msg_size, - migraphx_context_t ctx, - migraphx_shape_t output, - migraphx_arguments_t inputs); - -typedef migraphx_status (*migraphx_experimental_custom_op_compute_shape)(migraphx_shape_t out, - void* obj, - char* exception_msg, - size_t exception_msg_size, - migraphx_shapes_t inputs); - -typedef migraphx_status (*migraphx_experimental_custom_op_output_alias)(size_t* out, - size_t* out_size, - void* obj, - char* exception_msg, - size_t exception_msg_size, - migraphx_shapes_t inputs); - -typedef migraphx_status (*migraphx_experimental_custom_op_runs_on_offload_target)( - bool* out, void* obj, char* exception_msg, size_t exception_msg_size); - -typedef migraphx_status (*migraphx_experimental_custom_op_copy)(void** out, void* input); - -typedef migraphx_status (*migraphx_experimental_custom_op_delete)(void* input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_optimals_destroy(migraphx_optimals_t optimals); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_optimals_assign_to(migraphx_optimals_t output, - const_migraphx_optimals_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_optimals_create(migraphx_optimals_t* optimals, - const size_t* ptr, - size_t size); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimension_destroy(migraphx_dynamic_dimension_t dynamic_dimension); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_dynamic_dimension_assign_to( - migraphx_dynamic_dimension_t output, const_migraphx_dynamic_dimension_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_dynamic_dimension_create_min_max( - migraphx_dynamic_dimension_t* dynamic_dimension, size_t min, size_t max); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimension_create_min_max_optimals(migraphx_dynamic_dimension_t* dynamic_dimension, - size_t min, - size_t max, - migraphx_optimals_t optimals); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_dynamic_dimension_is_fixed( - bool* out, const_migraphx_dynamic_dimension_t dynamic_dimension); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimension_equal(bool* out, - const_migraphx_dynamic_dimension_t dynamic_dimension, - const_migraphx_dynamic_dimension_t x); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimensions_destroy(migraphx_dynamic_dimensions_t dynamic_dimensions); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_dynamic_dimensions_assign_to( - migraphx_dynamic_dimensions_t output, const_migraphx_dynamic_dimensions_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimensions_create(migraphx_dynamic_dimensions_t* dynamic_dimensions, - const const_migraphx_dynamic_dimension_t* ptr, - size_t size); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimensions_size(size_t* out, migraphx_dynamic_dimensions_t dynamic_dimensions); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_dynamic_dimensions_get(const_migraphx_dynamic_dimension_t* out, - migraphx_dynamic_dimensions_t dynamic_dimensions, - size_t idx); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_destroy(migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_assign_to(migraphx_shape_t output, - const_migraphx_shape_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_create(migraphx_shape_t* shape, - migraphx_shape_datatype_t type, - size_t* lengths, - size_t lengths_size); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_create_with_strides(migraphx_shape_t* shape, - migraphx_shape_datatype_t type, - size_t* lengths, - size_t lengths_size, - size_t* strides, - size_t strides_size); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_create_scalar(migraphx_shape_t* shape, - migraphx_shape_datatype_t type); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_create_dynamic(migraphx_shape_t* shape, - migraphx_shape_datatype_t type, - migraphx_dynamic_dimensions_t dims); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_lengths(const size_t** out, - size_t* out_size, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_strides(const size_t** out, - size_t* out_size, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_dyn_dims(migraphx_dynamic_dimensions_t* out, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_type(migraphx_shape_datatype_t* out, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_elements(size_t* out, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_bytes(size_t* out, const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_ndim(size_t* out, const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_equal(bool* out, - const_migraphx_shape_t shape, - const_migraphx_shape_t x); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_standard(bool* out, const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_dynamic(bool* out, const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shape_index(size_t* out, - const_migraphx_shape_t shape, - size_t i); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_destroy(migraphx_argument_t argument); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_assign_to(migraphx_argument_t output, - const_migraphx_argument_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_create(migraphx_argument_t* argument, - const_migraphx_shape_t shape, - void* buffer); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_create_empty(migraphx_argument_t* argument, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_shape(const_migraphx_shape_t* out, - const_migraphx_argument_t argument); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_buffer(char** out, - const_migraphx_argument_t argument); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_equal(bool* out, - const_migraphx_argument_t argument, - const_migraphx_argument_t x); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_argument_generate(migraphx_argument_t* out, - const_migraphx_shape_t s, - size_t seed); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_target_destroy(migraphx_target_t target); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_target_assign_to(migraphx_target_t output, - const_migraphx_target_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_target_create(migraphx_target_t* target, - const char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_parameter_shapes_destroy( - migraphx_program_parameter_shapes_t program_parameter_shapes); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_parameter_shapes_assign_to( - migraphx_program_parameter_shapes_t output, const_migraphx_program_parameter_shapes_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_parameter_shapes_size( - size_t* out, migraphx_program_parameter_shapes_t program_parameter_shapes); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_program_parameter_shapes_get(const_migraphx_shape_t* out, - migraphx_program_parameter_shapes_t program_parameter_shapes, - const char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_parameter_shapes_names( - const char** out, migraphx_program_parameter_shapes_t program_parameter_shapes); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_program_parameters_destroy(migraphx_program_parameters_t program_parameters); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_parameters_assign_to( - migraphx_program_parameters_t output, const_migraphx_program_parameters_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_program_parameters_create(migraphx_program_parameters_t* program_parameters); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_program_parameters_add(migraphx_program_parameters_t program_parameters, - const char* name, - const_migraphx_argument_t argument); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_arguments_destroy(migraphx_arguments_t arguments); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_arguments_assign_to(migraphx_arguments_t output, - const_migraphx_arguments_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_arguments_size(size_t* out, - migraphx_arguments_t arguments); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_arguments_get(const_migraphx_argument_t* out, - migraphx_arguments_t arguments, - size_t idx); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shapes_destroy(migraphx_shapes_t shapes); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shapes_assign_to(migraphx_shapes_t output, - const_migraphx_shapes_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shapes_size(size_t* out, migraphx_shapes_t shapes); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_shapes_get(const_migraphx_shape_t* out, - migraphx_shapes_t shapes, - size_t idx); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_instruction_destroy(migraphx_instruction_t instruction); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_instruction_assign_to(migraphx_instruction_t output, const_migraphx_instruction_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_instructions_destroy(migraphx_instructions_t instructions); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_instructions_assign_to( - migraphx_instructions_t output, const_migraphx_instructions_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_instructions_create( - migraphx_instructions_t* instructions, const const_migraphx_instruction_t* ptr, size_t size); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_modules_destroy(migraphx_modules_t modules); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_modules_assign_to(migraphx_modules_t output, - const_migraphx_modules_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_modules_create(migraphx_modules_t* modules, - migraphx_module_t* ptr, - size_t size); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_create(migraphx_module_t* module, char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_print(const_migraphx_module_t module); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_add_instruction(migraphx_instruction_t* out, - migraphx_module_t module, - migraphx_operation_t op, - migraphx_instructions_t args); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_module_add_instruction_with_mod_args(migraphx_instruction_t* out, - migraphx_module_t module, - migraphx_operation_t op, - migraphx_instructions_t args, - migraphx_modules_t module_refs); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_add_literal(migraphx_instruction_t* out, - migraphx_module_t module, - const_migraphx_shape_t shape, - const char* buffer); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_add_parameter(migraphx_instruction_t* out, - migraphx_module_t module, - const char* name, - const_migraphx_shape_t shape); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_add_return(migraphx_instruction_t* out, - migraphx_module_t module, - migraphx_instructions_t args); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_module_add_allocation(migraphx_instruction_t* out, - migraphx_module_t module, - const_migraphx_shape_t s); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_destroy(migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_assign_to(migraphx_program_t output, - const_migraphx_program_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_create(migraphx_program_t* program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_get_main_module(migraphx_module_t* out, - migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_create_module(migraphx_module_t* out, - migraphx_program_t program, - const char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_compile(migraphx_program_t program, - migraphx_target_t target, - migraphx_compile_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_get_parameter_shapes( - migraphx_program_parameter_shapes_t* out, migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_get_output_shapes(migraphx_shapes_t* out, - migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_print(const_migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_sort(migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_run(migraphx_arguments_t* out, - migraphx_program_t program, - migraphx_program_parameters_t params); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_run_async(migraphx_arguments_t* out, - migraphx_program_t program, - migraphx_program_parameters_t params, - void* s, - const char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_equal(bool* out, - const_migraphx_program_t program, - const_migraphx_program_t x); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_program_experimental_get_context( - migraphx_context_t* out, const_migraphx_program_t program); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_operation_destroy(migraphx_operation_t operation); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_operation_assign_to(migraphx_operation_t output, - const_migraphx_operation_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_operation_create(migraphx_operation_t* operation, - const char* name, - const char* attributes, - ...); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_operation_name(char* out, - size_t out_size, - migraphx_operation_t operation); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_load(migraphx_program_t* out, - const char* name, - migraphx_file_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_save(migraphx_program_t p, - const char* name, - migraphx_file_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_onnx_options_destroy(migraphx_onnx_options_t onnx_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_assign_to( - migraphx_onnx_options_t output, const_migraphx_onnx_options_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_onnx_options_create(migraphx_onnx_options_t* onnx_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_set_input_parameter_shape( - migraphx_onnx_options_t onnx_options, const char* name, size_t* dims, size_t dims_size); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_set_dyn_input_parameter_shape( - migraphx_onnx_options_t onnx_options, const char* name, migraphx_dynamic_dimensions_t dims); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_onnx_options_set_default_dim_value(migraphx_onnx_options_t onnx_options, size_t value); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_set_default_dyn_dim_value( - migraphx_onnx_options_t onnx_options, const_migraphx_dynamic_dimension_t dd); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_set_default_loop_iterations( - migraphx_onnx_options_t onnx_options, int64_t value); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_set_limit_loop_iterations( - migraphx_onnx_options_t onnx_options, int64_t value); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_onnx_options_set_external_data_path( - migraphx_onnx_options_t onnx_options, const char* external_data_path); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_file_options_destroy(migraphx_file_options_t file_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_file_options_assign_to( - migraphx_file_options_t output, const_migraphx_file_options_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_file_options_create(migraphx_file_options_t* file_options); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_file_options_set_file_format(migraphx_file_options_t file_options, const char* format); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_compile_options_destroy(migraphx_compile_options_t compile_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_compile_options_assign_to( - migraphx_compile_options_t output, const_migraphx_compile_options_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_compile_options_create(migraphx_compile_options_t* compile_options); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_compile_options_set_offload_copy(migraphx_compile_options_t compile_options, bool value); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_compile_options_set_fast_math(migraphx_compile_options_t compile_options, bool value); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_compile_options_set_exhaustive_tune_flag( - migraphx_compile_options_t compile_options, bool value); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_parse_onnx(migraphx_program_t* out, - const char* name, - migraphx_onnx_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_parse_onnx_buffer(migraphx_program_t* out, - const void* data, - size_t size, - migraphx_onnx_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_tf_options_destroy(migraphx_tf_options_t tf_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_tf_options_assign_to(migraphx_tf_options_t output, - const_migraphx_tf_options_t input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_tf_options_create(migraphx_tf_options_t* tf_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_tf_options_set_nhwc(migraphx_tf_options_t tf_options, - bool is_nhwc); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_tf_options_set_input_parameter_shape( - migraphx_tf_options_t tf_options, const char* name, size_t* dims, size_t dims_size); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_tf_options_set_default_dim_value(migraphx_tf_options_t tf_options, size_t value); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_tf_options_set_output_names( - migraphx_tf_options_t tf_options, const char** names, size_t names_size); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_parse_tf(migraphx_program_t* out, - const char* name, - migraphx_tf_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_op_names_destroy(migraphx_quantize_op_names_t quantize_op_names); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_op_names_assign_to( - migraphx_quantize_op_names_t output, const_migraphx_quantize_op_names_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_op_names_create(migraphx_quantize_op_names_t* quantize_op_names); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_op_names_add(migraphx_quantize_op_names_t quantize_op_names, const char* name); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_fp16_with_op_names(migraphx_program_t prog, migraphx_quantize_op_names_t name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_fp16(migraphx_program_t prog); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_int8_options_destroy(migraphx_quantize_int8_options_t quantize_int8_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_int8_options_assign_to( - migraphx_quantize_int8_options_t output, const_migraphx_quantize_int8_options_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_int8_options_create(migraphx_quantize_int8_options_t* quantize_int8_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_int8_options_add_op_name( - migraphx_quantize_int8_options_t quantize_int8_options, const char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_int8_options_add_calibration_data( - migraphx_quantize_int8_options_t quantize_int8_options, migraphx_program_parameters_t data); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_int8(migraphx_program_t prog, - migraphx_target_t target, - migraphx_quantize_int8_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_fp8_options_destroy(migraphx_quantize_fp8_options_t quantize_fp8_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_fp8_options_assign_to( - migraphx_quantize_fp8_options_t output, const_migraphx_quantize_fp8_options_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_quantize_fp8_options_create(migraphx_quantize_fp8_options_t* quantize_fp8_options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_fp8_options_add_calibration_data( - migraphx_quantize_fp8_options_t quantize_fp8_options, migraphx_program_parameters_t data); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_quantize_fp8(migraphx_program_t prog, - migraphx_target_t target, - migraphx_quantize_fp8_options_t options); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_context_finish(const_migraphx_context_t context); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_context_get_queue(void** out, - migraphx_context_t context); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_experimental_custom_op_destroy(migraphx_experimental_custom_op_t experimental_custom_op); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_experimental_custom_op_assign_to( - migraphx_experimental_custom_op_t output, const_migraphx_experimental_custom_op_t input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_experimental_custom_op_create(migraphx_experimental_custom_op_t* experimental_custom_op, - void* obj, - migraphx_experimental_custom_op_copy c, - migraphx_experimental_custom_op_delete d, - const char* obj_typename, - const char* name); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_experimental_custom_op_set_compute( - migraphx_experimental_custom_op_t obj, migraphx_experimental_custom_op_compute input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_experimental_custom_op_set_compute_shape( - migraphx_experimental_custom_op_t obj, migraphx_experimental_custom_op_compute_shape input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_experimental_custom_op_set_output_alias( - migraphx_experimental_custom_op_t obj, migraphx_experimental_custom_op_output_alias input); - -MIGRAPHX_C_EXPORT migraphx_status migraphx_experimental_custom_op_set_runs_on_offload_target( - migraphx_experimental_custom_op_t obj, - migraphx_experimental_custom_op_runs_on_offload_target input); - -MIGRAPHX_C_EXPORT migraphx_status -migraphx_experimental_custom_op_register(migraphx_experimental_custom_op_t experimental_custom_op); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/docker/rocm/migraphx/api/include/migraphx/migraphx.hpp b/docker/rocm/migraphx/api/include/migraphx/migraphx.hpp deleted file mode 100644 index 6c68f8764..000000000 --- a/docker/rocm/migraphx/api/include/migraphx/migraphx.hpp +++ /dev/null @@ -1,1589 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_API_RTGLIB_MIGRAPHX_HPP -#define MIGRAPHX_GUARD_API_RTGLIB_MIGRAPHX_HPP - -#include "migraphx.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -#ifndef DOXYGEN -inline namespace api { // NOLINT -#endif - -#ifdef __has_cpp_attribute -#if __has_cpp_attribute(deprecated) -#define MIGRAPHX_DEPRECATED(...) [[deprecated(__VA_ARGS__)]] -#endif -#endif - -#ifndef MIGRAPHX_DEPRECATED -#define MIGRAPHX_DEPRECATED(...) -#endif - -template -struct rank : rank -{ -}; - -template <> -struct rank<0> -{ -}; - -template -std::string compute_type_name() -{ - std::string name; -#if defined(_MSC_VER) && !defined(__clang__) - const char struct_name[] = "struct "; - const char class_name[] = "class "; - const char function_name[] = "compute_type_name<"; - const char parameter_name[] = ">(void)"; - const char cdecl_name[] = "__cdecl"; - - name = __FUNCSIG__; - - auto begin = name.find(function_name) + sizeof(function_name) - 1; - auto length = name.find(parameter_name) - begin; - name = name.substr(begin, length); - if(name.find(class_name) == 0) - name = name.substr(sizeof(class_name) - 1); - else if(name.find(struct_name) == 0) - name = name.substr(sizeof(struct_name) - 1); - begin = name.find(cdecl_name); - if(begin != std::string::npos) - name.erase(begin, sizeof(cdecl_name) - 1); -#else - const char parameter_name[] = "PrivateMigraphTypeNameProbe ="; // NOLINT - - name = __PRETTY_FUNCTION__; - - auto begin = name.find(parameter_name) + sizeof(parameter_name); -#if(defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7) - auto length = name.find_last_of(",") - begin; -#else - auto length = name.find_first_of("];", begin) - begin; -#endif - name = name.substr(begin, length); -#endif - return name; -} - -template -const std::string& get_type_name() -{ - static const std::string name = compute_type_name(); - return name; -} - -template -const std::string& get_type_name(const T&) -{ - return get_type_name(); -} - -template -T* make(F f, Ts&&... xs) -{ - T* result = nullptr; - auto e = f(&result, std::forward(xs)...); - if(e != migraphx_status_success) - throw std::runtime_error("Failed to call function"); - return result; -} - -template -void call(F f, Ts&&... xs) -{ - auto e = f(std::forward(xs)...); - if(e != migraphx_status_success) - throw std::runtime_error("Failed to call function"); -} - -template -struct iota_iterator -{ - Iterator index; - F f; - - using difference_type = std::ptrdiff_t; - using reference = decltype(f(std::declval())); - using value_type = typename std::remove_reference::type; - using pointer = typename std::add_pointer::type; - using iterator_category = std::input_iterator_tag; - - iota_iterator& operator+=(int n) - { - index += n; - return *this; - } - - iota_iterator& operator-=(int n) - { - index += n; - return *this; - } - - iota_iterator& operator++() - { - index++; - return *this; - } - - iota_iterator& operator--() - { - index--; - return *this; - } - - iota_iterator operator++(int) // NOLINT - { - iota_iterator it = *this; - index++; - return it; - } - - iota_iterator operator--(int) // NOLINT - { - iota_iterator it = *this; - index--; - return it; - } - // TODO: operator-> - reference operator*() const { return f(index); } - - friend iota_iterator operator+(iota_iterator x, iota_iterator y) - { - return iota_iterator(x.index + y.index, x.f); - } - - friend iota_iterator operator-(iota_iterator x, iota_iterator y) - { - return iota_iterator(x.index - y.index, x.f); - } - - friend bool operator==(iota_iterator x, iota_iterator y) { return x.index == y.index; } - - friend bool operator!=(iota_iterator x, iota_iterator y) { return x.index != y.index; } -}; - -template -struct array_base -{ - const Derived& derived() const { return static_cast(*this); } - - template - using value_type_t = decltype(std::declval()[0]); - - struct iterator_read - { - const Derived* self; - template - value_type_t operator()(size_t pidx) const - { - return (*self)[pidx]; - } - }; - - template - using iterator_t = iota_iterator; - - bool empty() const { return derived().size() == 0; } - - template - value_type_t front() const - { - return derived()[0]; - } - - template - value_type_t back() const - { - return derived()[derived().size() - 1]; - } - - template - iterator_t begin() const - { - return {0, {&derived()}}; - } - - template - iterator_t end() const - { - return {derived().size(), {&derived()}}; - } -}; - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wnon-template-friend" -#endif - -template -struct holder -{ - // Friend injection - friend auto migraphx_adl_handle_lookup(holder); - // Function left unimplemented since its only used in non-evaluated - // context - T get() const; -}; - -template -struct handle_lookup -{ - friend auto migraphx_adl_handle_lookup(holder) { return holder{}; } -}; - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - -template -using as_handle = - decltype(migraphx_adl_handle_lookup(holder>>{}) - .get()); - -struct own -{ -}; -struct borrow -{ -}; - -template -struct share -{ - share(std::shared_ptr p) : ptr(std::move(p)) {} - - template - std::shared_ptr alias(U* p) const - { - return std::shared_ptr{ptr, p}; - } - - private: - std::shared_ptr ptr; -}; - -template -struct handle_base : handle_lookup> -{ - using handle_type = T; - handle_base() : m_handle(nullptr) {} - template - void make_handle(F f, Ts&&... xs) - { - using type = typename std::remove_cv::type; - set_handle(make(f, std::forward(xs)...), own{}); - } - - const std::shared_ptr& get_handle() const { return m_handle; } - - T* get_handle_ptr() const - { - assert(m_handle != nullptr); - return get_handle().get(); - } - - template - void set_handle(U* ptr, own) - { - m_handle = std::shared_ptr{ptr, Deleter}; - } - - template - void set_handle(U* ptr, borrow) - { - m_handle = std::shared_ptr{ptr, [](U*) {}}; - } - - template - void set_handle(U* ptr, share b) - { - m_handle = std::shared_ptr{ptr, [b](U*) {}}; - } - - share share_handle() const { return {m_handle}; } - - template - void assign_to_handle(U* x) - { - Assigner(x, this->get_handle_ptr()); - } - - protected: - std::shared_ptr m_handle; -}; - -// NOLINTNEXTLINE -#define MIGRAPHX_HANDLE_CONSTRUCTOR(name) \ - template {}>::type> \ - name(HandleType* p, Lifetime lifetime) \ - { \ - this->set_handle(p, std::move(lifetime)); \ - } - -template -struct out_params -{ -}; - -template -struct interface_base : Base -{ - interface_base() : Base() {} - - protected: - template - static migraphx_status try_(F f, char* ex_msg = nullptr, size_t ex_msg_size = 0) // NOLINT - { - try - { - f(); - return migraphx_status_success; - } - catch(const std::exception& ex) - { - if(ex_msg) - { - std::strncpy(ex_msg, ex.what(), ex_msg_size); - ex_msg[ex_msg_size - 1] = '\0'; - } - return migraphx_status_unknown_error; - } - catch(...) - { - return migraphx_status_unknown_error; - } - } - - template - void make_interface(F f, T& obj, Ts&&... xs) - { - auto copy = [](void** out, void* input) { - return try_([&] { - T** y = reinterpret_cast(out); - T* x = reinterpret_cast(input); - assert(x != nullptr and y != nullptr and *y == nullptr); - // cppcheck-suppress useSmartPointer - *y = new T(*x); // NOLINT - }); - }; - auto del = [](void* input) { - return try_([&] { - T* x = reinterpret_cast(input); - delete x; // NOLINT - }); - }; - this->make_handle(f, &obj, copy, del, std::forward(xs)...); - } - - template - void set_fp(Setter setter, F pf, out_params<2>) - { - static F f = pf; - (void)f; // avoid warning on gcc - call(setter, - this->get_handle_ptr(), - [](auto out1, auto out2, void* obj, char* ex_msg, size_t ex_msg_size, auto... xs) - -> migraphx_status { - return try_([&] { call_cast_arg(rank<2>{}, f, out1, out2, obj, xs...); }, - ex_msg, - ex_msg_size); - }); - } - - template - void set_fp(Setter setter, F pf, out_params<1>) - { - static F f = pf; - (void)f; // avoid warning on gcc - call(setter, - this->get_handle_ptr(), - [](auto out, void* obj, char* ex_msg, size_t ex_msg_size, auto... xs) - -> migraphx_status { - return try_( - [&] { call_cast_arg(rank<1>{}, f, out, obj, xs...); }, ex_msg, ex_msg_size); - }); - } - - template - void set_fp(Setter setter, F pf, out_params<0>) - { - static F f = pf; - (void)f; // avoid warning on gcc - call(setter, - this->get_handle_ptr(), - [](void* obj, char* ex_msg, size_t ex_msg_size, auto... xs) -> migraphx_status { - return try_( - [&] { call_cast_arg(rank<0>{}, f, obj, xs...); }, ex_msg, ex_msg_size); - }); - } - - template - void set_auto_fp(Setter setter, F f, Out nums) - { - return set_fp( - setter, - [=](T& obj, auto out1, auto out2, auto... xs) { - auto_invoke(f, out1, out2, obj, auto_convert_param(rank<2>{}, xs)...); - }, - nums); - } - - struct no_out_arg - { - }; - - template {}>> - static void call_cast_arg(rank<0>, F f, X* obj, Xs... xs) - { - f(reinterpret_cast(obj), no_out_arg{}, no_out_arg{}, xs...); - } - - template {}>> - static void call_cast_arg(rank<1>, F f, R result, X* obj, Xs... xs) - { - f(*reinterpret_cast(obj), result, no_out_arg{}, xs...); - } - - template {}>> - static void call_cast_arg(rank<2>, F f, R1 result1, R2 result2, X* obj, Xs... xs) - { - f(*reinterpret_cast(obj), result1, result2, xs...); - } - - template - void auto_invoke(F f, T1* out1, T2* out2, Ts&&... xs) - { - auto_assign(rank<2>{}, out1, out2, f(std::forward(xs)...)); - } - - template - void auto_invoke(F f, T* out, no_out_arg, Ts&&... xs) - { - auto_assign(rank<1>{}, out, f(std::forward(xs)...)); - } - - template - void auto_invoke(F f, no_out_arg, no_out_arg, Ts&&... xs) - { - f(std::forward(xs)...); - } - - template {} or std::is_enum{}>> - T auto_convert_param(rank<0>, T x) - { - return x; - } - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - template - auto auto_convert_param(rank<1>, T x) -> decltype(as_handle{x}) - { - return as_handle{x}; - } -#pragma GCC diagnostic pop - - template - auto auto_convert_param(rank<2>, T x) -> decltype(as_handle{x, borrow{}}) - { - return as_handle{x, borrow{}}; - } - - template - void auto_assign(rank<0>, T* out, U x) - { - *out = x; - } - - template - auto auto_assign(rank<1>, T* out, U x) -> decltype(x.assign_to_handle(out)) - { - x.assign_to_handle(out); - } - - template {}>> - auto auto_assign(rank<2>, T1* out_ptr, T2* out_size, U x) - { - *out_size = std::min(*out_size, x.size()); - std::copy_n(x.begin(), *out_size, out_ptr); - } -}; - -// NOLINTNEXTLINE -#define MIGRAPHX_INTERFACE_LIFT(n_out, T, prefix, name) \ - this->set_auto_fp( \ - &migraphx_##prefix##_set_##name, \ - [](T& x, auto... xs) { return x.name(xs...); }, \ - out_params{}) - -template -using require_interface = - std::enable_if_t{} and not std::is_same{} and - std::is_copy_constructible{} and std::is_final{}>; - -#ifdef DOXYGEN -#define MIGRAPHX_DETAIL_HANDLE_BASE(name, const_) handle_base<> -#else -#define MIGRAPHX_DETAIL_HANDLE_BASE(name, const_) \ - handle_base -#endif -// NOLINTNEXTLINE -#define MIGRAPHX_HANDLE_BASE(name) MIGRAPHX_DETAIL_HANDLE_BASE(name, ) -// NOLINTNEXTLINE -#define MIGRAPHX_CONST_HANDLE_BASE(name) MIGRAPHX_DETAIL_HANDLE_BASE(name, const) - -/** - * Container to hold optimal dynamic dimension values. - */ -struct optimals : MIGRAPHX_HANDLE_BASE(optimals) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(optimals) - - optimals(std::initializer_list init_list) - { - this->make_handle(&migraphx_optimals_create, init_list.begin(), init_list.size()); - } -}; - -/** - * @brief Dynamic dimension object. - * @details minimum, maximum, and optimal dimensions - */ -struct dynamic_dimension : MIGRAPHX_CONST_HANDLE_BASE(dynamic_dimension) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(dynamic_dimension) - - dynamic_dimension(size_t min, size_t max) - { - this->make_handle(&migraphx_dynamic_dimension_create_min_max, min, max); - } - - dynamic_dimension(size_t min, size_t max, const optimals& opts) - { - this->make_handle( - &migraphx_dynamic_dimension_create_min_max_optimals, min, max, opts.get_handle_ptr()); - } - - bool is_fixed() const - { - bool result = false; - call(&migraphx_dynamic_dimension_is_fixed, &result, this->get_handle_ptr()); - return result; - } - - friend bool operator==(const dynamic_dimension& x, const dynamic_dimension& y) - { - bool pout; - call(&migraphx_dynamic_dimension_equal, &pout, x.get_handle_ptr(), y.get_handle_ptr()); - return pout; - } - - friend bool operator!=(const dynamic_dimension& x, const dynamic_dimension& y) - { - return not(x == y); - } -}; - -/** - * Container to hold dynamic_dimension objects. - */ -struct dynamic_dimensions : MIGRAPHX_HANDLE_BASE(dynamic_dimensions) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(dynamic_dimensions) - - template - dynamic_dimensions(Ts... xs) - { - std::array a{xs.get_handle_ptr()...}; - this->make_handle(&migraphx_dynamic_dimensions_create, a.data(), a.size()); - } - - size_t size() const - { - size_t pout; - call(&migraphx_dynamic_dimensions_size, &pout, this->get_handle_ptr()); - return pout; - } - - dynamic_dimension operator[](size_t pidx) const - { - const_migraphx_dynamic_dimension_t pout; - call(&migraphx_dynamic_dimensions_get, &pout, this->get_handle_ptr(), pidx); - return {pout, this->share_handle()}; - } -}; - -/** - * @brief Describe shape of tensor - * @details A shape consists of a data type, lengths of multi-dimension tensor, and strides - */ -struct shape : MIGRAPHX_CONST_HANDLE_BASE(shape) -{ - shape() {} - - MIGRAPHX_DEPRECATED("Contructor without lifetime annotation is deprecated.") - shape(const migraphx_shape* p) { this->set_handle(p, borrow{}); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(shape) - - /// Construct a scalar shape - shape(migraphx_shape_datatype_t type) - { - this->make_handle(&migraphx_shape_create_scalar, type); - } - - /// Construct a shape with its type and lengths. The strides are - /// automatically computed assumming a packed layout. - shape(migraphx_shape_datatype_t type, std::vector plengths) - { - this->make_handle(&migraphx_shape_create, type, plengths.data(), plengths.size()); - } - - // Force all calls of the format `shape( type_t, { size_t compatibles } )` to map to - // shape(type_t, std::vector l) - shape(migraphx_shape_datatype_t t, std::initializer_list d) - : shape::shape(t, std::vector{d.begin(), d.end()}) - { - } - - shape(migraphx_shape_datatype_t type, - std::vector plengths, - std::vector pstrides) - { - this->make_handle(&migraphx_shape_create_with_strides, - type, - plengths.data(), - plengths.size(), - pstrides.data(), - pstrides.size()); - } - - shape(migraphx_shape_datatype_t type, const dynamic_dimensions& dyn_dims) - { - this->make_handle(&migraphx_shape_create_dynamic, type, dyn_dims.get_handle_ptr()); - } - - std::vector lengths() const - { - const size_t* pout; - size_t pout_size; - call(&migraphx_shape_lengths, &pout, &pout_size, this->get_handle_ptr()); - return {pout, pout + pout_size}; - } - - std::vector strides() const - { - const size_t* pout; - size_t pout_size; - call(&migraphx_shape_strides, &pout, &pout_size, this->get_handle_ptr()); - return {pout, pout + pout_size}; - } - - /// Get the dynamic dimensions of the shape - dynamic_dimensions dyn_dims() const - { - migraphx_dynamic_dimensions_t pout; - call(&migraphx_shape_dyn_dims, &pout, this->get_handle_ptr()); - return {pout, own{}}; - } - - migraphx_shape_datatype_t type() const - { - migraphx_shape_datatype_t pout; - call(&migraphx_shape_type, &pout, this->get_handle_ptr()); - return pout; - } - - size_t elements() const - { - size_t pout; - call(&migraphx_shape_elements, &pout, this->get_handle_ptr()); - return pout; - } - - size_t bytes() const - { - size_t pout; - call(&migraphx_shape_bytes, &pout, this->get_handle_ptr()); - return pout; - } - - bool standard() const - { - bool result = false; - call(&migraphx_shape_standard, &result, this->get_handle_ptr()); - return result; - } - - /// Is the shape dynamic - bool dynamic() const - { - bool result = false; - call(&migraphx_shape_dynamic, &result, this->get_handle_ptr()); - return result; - } - - // map element index to space index - size_t index(size_t i) const - { - size_t result; - call(&migraphx_shape_index, &result, this->get_handle_ptr(), i); - return result; - } - - friend bool operator==(const shape& px, const shape& py) - { - bool pout; - call(&migraphx_shape_equal, &pout, px.get_handle_ptr(), py.get_handle_ptr()); - return pout; - } - - friend bool operator!=(const shape& px, const shape& py) { return not(px == py); } -}; - -/** - * @brief Arguments to be passed to an migraphx arguments - * - * An `argument` represents a raw buffer of data with a shape. - * - */ -struct argument : MIGRAPHX_CONST_HANDLE_BASE(argument) -{ - argument() {} - - MIGRAPHX_HANDLE_CONSTRUCTOR(argument) - - MIGRAPHX_DEPRECATED("Contructor without lifetime annotation is deprecated.") - argument(const migraphx_argument* p) { this->set_handle(p, borrow{}); } - - argument(shape pshape) - { - this->make_handle(&migraphx_argument_create_empty, pshape.get_handle_ptr()); - } - - argument(shape pshape, void* pbuffer) - { - this->make_handle(&migraphx_argument_create, pshape.get_handle_ptr(), pbuffer); - } - - shape get_shape() const - { - const_migraphx_shape_t pout; - call(&migraphx_argument_shape, &pout, this->get_handle_ptr()); - return {pout, this->share_handle()}; - } - - char* data() const - { - char* pout; - call(&migraphx_argument_buffer, &pout, this->get_handle_ptr()); - return pout; - } - - template - std::vector as_vector() const - { - auto ss = this->get_shape(); - auto num_elements = ss.elements(); - std::vector res(num_elements); - T* buffer_ptr = reinterpret_cast(this->data()); - for(size_t i = 0; i < num_elements; i++) - { - res[i] = buffer_ptr[ss.index(i)]; - } - return res; - } - - /// Generate an argument using random data - static argument generate(shape ps, size_t pseed = 0) - { - return {make(&migraphx_argument_generate, ps.get_handle_ptr(), pseed), - own{}}; - } - - friend bool operator==(const argument& px, const argument& py) - { - bool pout; - call(&migraphx_argument_equal, &pout, px.get_handle_ptr(), py.get_handle_ptr()); - return pout; - } - - friend bool operator!=(const argument& px, const argument& py) { return not(px == py); } -}; - -/// A target for compilation -struct target : MIGRAPHX_HANDLE_BASE(target) -{ - target() {} - - MIGRAPHX_HANDLE_CONSTRUCTOR(target) - - /// Construct a target from its name - target(const char* name) { this->make_handle(&migraphx_target_create, name); } -}; - -struct program_parameter_shapes : MIGRAPHX_HANDLE_BASE(program_parameter_shapes) -{ - program_parameter_shapes() {} - - MIGRAPHX_HANDLE_CONSTRUCTOR(program_parameter_shapes) - - size_t size() const - { - size_t pout; - call(&migraphx_program_parameter_shapes_size, &pout, this->get_handle_ptr()); - return pout; - } - - shape operator[](const char* pname) const - { - const_migraphx_shape_t pout; - call(&migraphx_program_parameter_shapes_get, &pout, this->get_handle_ptr(), pname); - return {pout, this->share_handle()}; - } - - std::vector names() const - { - std::vector result(this->size()); - if(not result.empty()) - { - call(&migraphx_program_parameter_shapes_names, result.data(), this->get_handle_ptr()); - } - return result; - } -}; - -/// A class to construct the inputs parameters for a program -struct program_parameters : MIGRAPHX_HANDLE_BASE(program_parameters) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(program_parameters) - - MIGRAPHX_DEPRECATED("Contructor without lifetime annotation is deprecated.") - program_parameters(migraphx_program_parameters* p) { this->set_handle(p, borrow{}); } - - program_parameters() { this->make_handle(&migraphx_program_parameters_create); } - - /// Construct the parameters from initializer_list - program_parameters(std::initializer_list> l) - { - this->make_handle(&migraphx_program_parameters_create); - for(auto&& p : l) - this->add(p.first.c_str(), p.second); - } - - /// Add a new parameter - void add(const char* pname, const argument& pargument) const - { - call(&migraphx_program_parameters_add, - this->get_handle_ptr(), - pname, - pargument.get_handle_ptr()); - } -}; - -struct arguments : MIGRAPHX_HANDLE_BASE(arguments), array_base -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(arguments) - - size_t size() const - { - size_t pout; - call(&migraphx_arguments_size, &pout, this->get_handle_ptr()); - return pout; - } - - argument operator[](size_t pidx) const - { - const_migraphx_argument_t pout; - call(&migraphx_arguments_get, &pout, this->get_handle_ptr(), pidx); - return {pout, this->share_handle()}; - } -}; - -struct shapes : MIGRAPHX_HANDLE_BASE(shapes), array_base -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(shapes) - - size_t size() const - { - size_t pout; - call(&migraphx_shapes_size, &pout, this->get_handle_ptr()); - return pout; - } - - shape operator[](size_t pidx) const - { - const_migraphx_shape_t pout; - call(&migraphx_shapes_get, &pout, this->get_handle_ptr(), pidx); - return {pout, this->share_handle()}; - } -}; - -struct operation : MIGRAPHX_HANDLE_BASE(operation) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(operation) - - template - operation(const char* name, const char* attributes = nullptr, Ts... xs) - { - this->make_handle(&migraphx_operation_create, name, attributes, xs...); - } - - std::string name() - { - std::array out_name; - call(&migraphx_operation_name, out_name.data(), 1024, this->get_handle_ptr()); - return {out_name.data()}; - } -}; - -struct instruction : MIGRAPHX_CONST_HANDLE_BASE(instruction) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(instruction) -}; - -struct instructions : MIGRAPHX_HANDLE_BASE(instructions) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(instructions) - - template - instructions(Ts... xs) - { - std::array a{xs.get_handle_ptr()...}; - this->make_handle(&migraphx_instructions_create, a.data(), a.size()); - } -}; - -struct module; - -struct modules : MIGRAPHX_HANDLE_BASE(modules) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(modules) - - template - modules(Ts... xs) - { - std::array a = {xs.get_handle_ptr()...}; - this->make_handle(&migraphx_modules_create, a.data(), a.size()); - } -}; - -struct module -{ - MIGRAPHX_DEPRECATED("Constructor without lifetime annotation is deprecated.") - module(migraphx_module* m) :mm(std::shared_ptr(), m) {} - - module(migraphx_module* m, borrow) :mm(std::shared_ptr(), m) {} - - template - module(migraphx_module* m, share b) : mm(b.alias(m)) - { - } - - void print() const { call(&migraphx_module_print, mm.get()); } - - instruction add_instruction(const migraphx::operation& op, const migraphx::instructions& args) - { - migraphx_instruction_t op_ins; - call(&migraphx_module_add_instruction, - &op_ins, - mm.get(), - op.get_handle_ptr(), - args.get_handle_ptr()); - return instruction(op_ins, own{}); - } - - instruction add_instruction(const migraphx::operation& op, - const migraphx::instructions& args, - const migraphx::modules& module_args) - { - migraphx_instruction_t op_ins; - call(&migraphx_module_add_instruction_with_mod_args, - &op_ins, - mm.get(), - op.get_handle_ptr(), - args.get_handle_ptr(), - module_args.get_handle_ptr()); - return instruction(op_ins, own{}); - } - - template - instruction add_literal(const migraphx::shape& s, T* buffer) - { - migraphx_instruction_t literal_ins; - const auto* buffer_ptr = reinterpret_cast(buffer); - call(&migraphx_module_add_literal, &literal_ins, mm.get(), s.get_handle_ptr(), buffer_ptr); - return instruction(literal_ins, own{}); - } - - instruction add_parameter(const std::string& name, shape s) - { - migraphx_instruction_t param_ins; - call( - &migraphx_module_add_parameter, ¶m_ins, mm.get(), name.c_str(), s.get_handle_ptr()); - return instruction(param_ins, own{}); - } - - instruction add_return(const migraphx::instructions& args) - { - migraphx_instruction_t ret_ins; - call(&migraphx_module_add_return, &ret_ins, mm.get(), args.get_handle_ptr()); - return instruction(ret_ins, own{}); - } - - instruction add_allocation(const migraphx::shape& s) - { - migraphx_instruction_t ret_ins; - call(&migraphx_module_add_allocation, &ret_ins, mm.get(), s.get_handle_ptr()); - return instruction(ret_ins, own{}); - } - - migraphx_module_t get_handle_ptr() const { return mm.get(); } - - private: - std::shared_ptr mm; -}; - -struct context : handle_lookup -{ - context(migraphx_context* p, borrow) : ctx(std::shared_ptr(), p) {} - - template - context(migraphx_context* p, share b) : ctx(b.alias(p)) - { - } - - void finish() const { call(&migraphx_context_finish, ctx.get()); } - - template - T get_queue() - { - void* out; - call(&migraphx_context_get_queue, &out, ctx.get()); - // TODO: check type here - return reinterpret_cast(out); - } - - private: - std::shared_ptr ctx; -}; - -struct compile_options : MIGRAPHX_HANDLE_BASE(compile_options) -{ - compile_options() { this->make_handle(&migraphx_compile_options_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(compile_options) - - /// For targets with offloaded memory(such as the gpu), this will insert - /// instructions during compilation to copy the input parameters to the - /// offloaded memory and to copy the final result from the offloaded - /// memory back to main memory. - void set_offload_copy(bool value = true) - { - call(&migraphx_compile_options_set_offload_copy, this->get_handle_ptr(), value); - } - - /// Optimize math functions to use faster approximate versions. There may - /// be slight accuracy degredation when enabled. - void set_fast_math(bool value = true) - { - call(&migraphx_compile_options_set_fast_math, this->get_handle_ptr(), value); - } - - /// Set or un-set exhaustive search to find fastest kernel - void set_exhaustive_tune_flag(bool value = true) - { - call(&migraphx_compile_options_set_exhaustive_tune_flag, this->get_handle_ptr(), value); - } -}; - -/// A program represents the all computation graphs to be compiled and executed -struct program : MIGRAPHX_HANDLE_BASE(program) -{ - program() { this->make_handle(&migraphx_program_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(program) - - /// Compile the program for a specific target to be ran on - void compile(const target& ptarget, const compile_options& poptions) const - { - call(&migraphx_program_compile, - this->get_handle_ptr(), - ptarget.get_handle_ptr(), - poptions.get_handle_ptr()); - } - - /// Compile the program for a specific target to be ran on - void compile(const target& ptarget) const - { - call(&migraphx_program_compile, - this->get_handle_ptr(), - ptarget.get_handle_ptr(), - migraphx::compile_options{}.get_handle_ptr()); - } - - /// Return the shapes for the input parameters - program_parameter_shapes get_parameter_shapes() const - { - migraphx_program_parameter_shapes_t pout; - call(&migraphx_program_get_parameter_shapes, &pout, this->get_handle_ptr()); - return program_parameter_shapes(pout, own{}); - } - - /// Get the shapes of all the outputs returned by this program - shapes get_output_shapes() const - { - migraphx_shapes_t pout; - call(&migraphx_program_get_output_shapes, &pout, this->get_handle_ptr()); - return shapes(pout, own{}); - } - - /// Run the program using the inputs passed in - arguments eval(const program_parameters& pparams) const - { - migraphx_arguments_t pout; - call(&migraphx_program_run, &pout, this->get_handle_ptr(), pparams.get_handle_ptr()); - return arguments(pout, own{}); - } - - template - /// Overloaded to allow for execution_environment input - arguments run_async(const program_parameters& pparams, Stream* s) const - { - migraphx_arguments_t pout; - call(&migraphx_program_run_async, - &pout, - this->get_handle_ptr(), - pparams.get_handle_ptr(), - s, - get_type_name().c_str()); - return arguments(pout, own{}); - } - - void print() const { call(&migraphx_program_print, this->get_handle_ptr()); } - - program sort() - { - call(&migraphx_program_sort, this->get_handle_ptr()); - return *this; - } - - friend bool operator==(const program& px, const program& py) - { - bool pout; - call(&migraphx_program_equal, &pout, px.get_handle_ptr(), py.get_handle_ptr()); - return pout; - } - - module get_main_module() - { - migraphx_module_t p_modu; - call(&migraphx_program_get_main_module, &p_modu, this->get_handle_ptr()); - return module{p_modu, this->share_handle()}; - } - - context experimental_get_context() - { - migraphx_context_t ctx; - call(&migraphx_program_experimental_get_context, &ctx, this->get_handle_ptr()); - return context{ctx, this->share_handle()}; - } - - module create_module(const std::string& name) - { - migraphx_module_t p_modu; - call(&migraphx_program_create_module, &p_modu, this->get_handle_ptr(), name.data()); - return module{p_modu, this->share_handle()}; - } - - friend bool operator!=(const program& px, const program& py) { return not(px == py); } -}; - -// options for migraphx file format options -struct file_options : MIGRAPHX_HANDLE_BASE(file_options) -{ - MIGRAPHX_HANDLE_CONSTRUCTOR(file_options) - file_options() { this->make_handle(&migraphx_file_options_create); } - - // set file format - void set_file_format(const char* format) - { - call(&migraphx_file_options_set_file_format, this->get_handle_ptr(), format); - } -}; - -/// Load a saved migraphx program from a file -inline program load(const char* filename, const file_options& options) -{ - return program(make(&migraphx_load, filename, options.get_handle_ptr()), - own{}); -} - -/// Load a saved migraphx program from a file -inline program load(const char* filename) -{ - return program( - make(&migraphx_load, filename, migraphx::file_options{}.get_handle_ptr()), - own{}); -} - -/// Save a program to a file -inline void save(const program& p, const char* filename, const file_options& options) -{ - call(&migraphx_save, p.get_handle_ptr(), filename, options.get_handle_ptr()); -} - -/// Save a program to a file -inline void save(const program& p, const char* filename) -{ - call(&migraphx_save, p.get_handle_ptr(), filename, migraphx::file_options{}.get_handle_ptr()); -} - -/// Options for parsing onnx options -struct onnx_options : MIGRAPHX_HANDLE_BASE(onnx_options) -{ - onnx_options() { this->make_handle(&migraphx_onnx_options_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(onnx_options) - - /// Make onnx parser treat an inputs with a certain dimensions - void set_input_parameter_shape(const std::string& name, std::vector dim) - { - call(&migraphx_onnx_options_set_input_parameter_shape, - this->get_handle_ptr(), - name.c_str(), - dim.data(), - dim.size()); - } - - void set_dyn_input_parameter_shape(const std::string& name, const dynamic_dimensions& dyn_dims) - { - call(&migraphx_onnx_options_set_dyn_input_parameter_shape, - this->get_handle_ptr(), - name.c_str(), - dyn_dims.get_handle_ptr()); - } - - /// When there is a dimension parameter, then use this default value - void set_default_dim_value(unsigned int value) - { - call(&migraphx_onnx_options_set_default_dim_value, this->get_handle_ptr(), value); - } - - void set_default_dyn_dim_value(const dynamic_dimension& dd) - { - call(&migraphx_onnx_options_set_default_dyn_dim_value, - this->get_handle_ptr(), - dd.get_handle_ptr()); - } - - /// Set default max iteration number for the loop operator - void set_default_loop_iterations(int64_t value) - { - call(&migraphx_onnx_options_set_default_loop_iterations, this->get_handle_ptr(), value); - } - - /// Set max iteration limit for the loop operator - void set_limit_loop_iterations(int64_t value) - { - call(&migraphx_onnx_options_set_limit_loop_iterations, this->get_handle_ptr(), value); - } - - /// Set absolute path for external data files - void set_external_data_path(const std::string& external_data_path) - { - call(&migraphx_onnx_options_set_external_data_path, - this->get_handle_ptr(), - external_data_path.c_str()); - } -}; - -/// Parse an onnx file into a migraphx program -inline program parse_onnx(const char* filename, const migraphx::onnx_options& options) -{ - return program(make(&migraphx_parse_onnx, filename, options.get_handle_ptr()), - own{}); -} - -/// Parse an onnx file into a migraphx program -inline program parse_onnx(const char* filename) -{ - migraphx::onnx_options options; - return program(make(&migraphx_parse_onnx, filename, options.get_handle_ptr()), - own{}); -} - -/// Parse a buffer of memory as an onnx file -inline program -parse_onnx_buffer(const void* data, size_t size, const migraphx::onnx_options& options) -{ - return program( - make(&migraphx_parse_onnx_buffer, data, size, options.get_handle_ptr()), - own{}); -} - -/// Parse a buffer of memory as an onnx file -inline program parse_onnx_buffer(const void* data, size_t size) -{ - migraphx::onnx_options options; - return program( - make(&migraphx_parse_onnx_buffer, data, size, options.get_handle_ptr()), - own{}); -} - -/// Parse a buffer of memory as an onnx file -inline program parse_onnx_buffer(const std::string& buffer, const migraphx::onnx_options& options) -{ - return program( - make( - &migraphx_parse_onnx_buffer, buffer.data(), buffer.size(), options.get_handle_ptr()), - own{}); -} - -/// Parse a buffer of memory as an onnx file -inline program parse_onnx_buffer(const std::string& buffer) -{ - migraphx::onnx_options options; - return program( - make( - &migraphx_parse_onnx_buffer, buffer.data(), buffer.size(), options.get_handle_ptr()), - own{}); -} - -/// Options for parsing tf options -struct tf_options : MIGRAPHX_HANDLE_BASE(tf_options) -{ - tf_options() { this->make_handle(&migraphx_tf_options_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(tf_options) - - /// Make tf parser treat an inputs with a certain dimensions - void set_input_parameter_shape(const std::string& name, std::vector dim) - { - call(&migraphx_tf_options_set_input_parameter_shape, - this->get_handle_ptr(), - name.c_str(), - dim.data(), - dim.size()); - } - - /// Change data layout to NHWC (default is NCHW) - void set_nhwc(bool is_nhwc = true) - { - call(&migraphx_tf_options_set_nhwc, this->get_handle_ptr(), is_nhwc); - } - - /// When there is a dimension parameter, then use this default value - void set_default_dim_value(unsigned int value) - { - call(&migraphx_tf_options_set_default_dim_value, this->get_handle_ptr(), value); - } - - /// Set output node names to return specific outputs from graph - void set_output_names(std::vector names) - { - call(&migraphx_tf_options_set_output_names, - this->get_handle_ptr(), - names.data(), - names.size()); - } -}; - -/// Parse a tf file into a migraphx program -inline program parse_tf(const char* filename, const migraphx::tf_options& options) -{ - return program(make(&migraphx_parse_tf, filename, options.get_handle_ptr()), - own{}); -} - -/// Parse a tf file into a migraphx program -inline program parse_tf(const char* filename) -{ - migraphx::tf_options options; - return program(make(&migraphx_parse_tf, filename, options.get_handle_ptr()), - own{}); -} - -struct quantize_op_names : MIGRAPHX_HANDLE_BASE(quantize_op_names) -{ - quantize_op_names() { this->make_handle(&migraphx_quantize_op_names_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(quantize_op_names) - - void add(const std::string& name) - { - call(&migraphx_quantize_op_names_add, this->get_handle_ptr(), name.c_str()); - } -}; - -/// Quantize program to use fp16 -inline void quantize_fp16(const program& prog, const quantize_op_names& names) -{ - call(&migraphx_quantize_fp16_with_op_names, prog.get_handle_ptr(), names.get_handle_ptr()); -} - -/// Quantize program to use fp16 -inline void quantize_fp16(const program& prog) -{ - call(&migraphx_quantize_fp16, prog.get_handle_ptr()); -} - -/// Options to be passed when quantizing for int8 -struct quantize_int8_options : MIGRAPHX_HANDLE_BASE(quantize_int8_options) -{ - quantize_int8_options() { this->make_handle(&migraphx_quantize_int8_options_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(quantize_int8_options) - - /// Add an operator that should be quantized - void add_op_name(const std::string& name) - { - call(&migraphx_quantize_int8_options_add_op_name, this->get_handle_ptr(), name.c_str()); - } - - /// Add calibrartion data to be used for quantizing - void add_calibration_data(const program_parameters& pp) - { - call(&migraphx_quantize_int8_options_add_calibration_data, - this->get_handle_ptr(), - pp.get_handle_ptr()); - } -}; - -/// Quantize program to use int8 -inline void -quantize_int8(const program& prog, const target& ptarget, const quantize_int8_options& options) -{ - call(&migraphx_quantize_int8, - prog.get_handle_ptr(), - ptarget.get_handle_ptr(), - options.get_handle_ptr()); -} - -/// Options to be passed when quantizing for int8 -struct quantize_fp8_options : MIGRAPHX_HANDLE_BASE(quantize_fp8_options) -{ - quantize_fp8_options() { this->make_handle(&migraphx_quantize_fp8_options_create); } - - MIGRAPHX_HANDLE_CONSTRUCTOR(quantize_fp8_options) - /// Add calibrartion data to be used for quantizing - void add_calibration_data(const program_parameters& pp) - { - call(&migraphx_quantize_fp8_options_add_calibration_data, - this->get_handle_ptr(), - pp.get_handle_ptr()); - } -}; - -/// Quantize program to use fp8 -inline void -quantize_fp8(const program& prog, const target& ptarget, const quantize_fp8_options& options) -{ - call(&migraphx_quantize_fp8, - prog.get_handle_ptr(), - ptarget.get_handle_ptr(), - options.get_handle_ptr()); -} - -struct experimental_custom_op_base -{ - experimental_custom_op_base() = default; - experimental_custom_op_base(const experimental_custom_op_base&) = default; - experimental_custom_op_base& operator=(const experimental_custom_op_base&) = default; - virtual ~experimental_custom_op_base() = default; - - virtual std::string name() const = 0; - virtual argument compute(context ctx, shape output, arguments inputs) const = 0; - virtual shape compute_shape(shapes inputs) const = 0; - virtual std::vector output_alias(shapes) const { return {}; } - // TODO: Return target string instead of bool - virtual bool runs_on_offload_target() const = 0; -}; - -struct experimental_custom_op : interface_base -{ - template - experimental_custom_op(T& obj) - { - this->make_interface(&migraphx_experimental_custom_op_create, - obj, - get_type_name(obj).c_str(), - obj.name().c_str()); - MIGRAPHX_INTERFACE_LIFT(1, T, experimental_custom_op, compute_shape); - MIGRAPHX_INTERFACE_LIFT(1, T, experimental_custom_op, compute); - MIGRAPHX_INTERFACE_LIFT(2, T, experimental_custom_op, output_alias); - MIGRAPHX_INTERFACE_LIFT(1, T, experimental_custom_op, runs_on_offload_target); - } - - void register_op() { call(&migraphx_experimental_custom_op_register, this->get_handle_ptr()); } -}; - -template > -void register_experimental_custom_op(T& obj) -{ - experimental_custom_op op{obj}; - op.register_op(); -} - -#ifndef DOXYGEN -} // namespace api -#endif -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/api/migraphx.py b/docker/rocm/migraphx/api/migraphx.py deleted file mode 100644 index 17d248219..000000000 --- a/docker/rocm/migraphx/api/migraphx.py +++ /dev/null @@ -1,510 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### -import api - - -def bad_param_error(msg): - return 'MIGRAPHX_THROW(migraphx_status_bad_param, "{}")'.format(msg) - - -api.error_type = 'migraphx_status' -api.success_type = 'migraphx_status_success' -api.try_wrap = 'migraphx::try_' -api.bad_param_error = bad_param_error - - -@api.cwrap('migraphx::shape::type_t') -def shape_type_wrap(p): - if p.returns: - p.add_param('migraphx_shape_datatype_t *') - p.bad_param('${name} == nullptr', 'Null pointer') - p.write = ['*${name} = migraphx::to_shape_type(${result})'] - else: - p.add_param('migraphx_shape_datatype_t') - p.read = 'migraphx::to_shape_type(${name})' - - -def auto_handle(*args, **kwargs): - def with_handle(f): - return api.handle('migraphx_' + f.__name__, 'migraphx::' + f.__name__, - *args, **kwargs)(f) - - return with_handle - - -@api.handle('migraphx_optimals', 'std::set') -def optimals(h): - h.constructor('create', - api.params(ptr='const size_t*', size='size_t'), - fname='migraphx::make_set') - - -@api.handle('migraphx_dynamic_dimension', 'migraphx::shape::dynamic_dimension') -def dynamic_dimension(h): - h.constructor('create_min_max', api.params(min='size_t', max='size_t')) - h.constructor( - 'create_min_max_optimals', - api.params(min='size_t', max='size_t', optimals='std::set')) - h.method('is_fixed', returns='bool', const=True) - h.method('equal', - api.params(x='const migraphx::shape::dynamic_dimension&'), - invoke='migraphx::equal($@)', - returns='bool', - const=True) - - -@api.handle('migraphx_dynamic_dimensions', - 'std::vector') -def dynamic_dimensions(h): - h.constructor( - 'create', - api.params(ptr='const const_migraphx_dynamic_dimension_t*', - size='size_t'), - fname='migraphx::to_obj_vector') - h.method('size', returns='size_t') - h.method('get', - api.params(idx='size_t'), - fname='at', - cpp_name='operator[]', - returns='const migraphx::shape::dynamic_dimension&') - - -@auto_handle() -def shape(h): - h.constructor( - 'create', - api.params(type='migraphx::shape::type_t', - lengths='std::vector')) - h.constructor( - 'create_with_strides', - api.params(type='migraphx::shape::type_t', - lengths='std::vector', - strides='std::vector')) - h.constructor('create_scalar', api.params(type='migraphx::shape::type_t')) - h.constructor( - 'create_dynamic', - api.params(type='migraphx::shape::type_t', - dims='std::vector')) - h.method('lengths', - fname='lens', - returns='const std::vector&', - const=True) - h.method('strides', returns='const std::vector&', const=True) - h.method('dyn_dims', - returns='std::vector', - const=True) - h.method('type', returns='migraphx::shape::type_t', const=True) - h.method('elements', returns='size_t', const=True) - h.method('bytes', returns='size_t', const=True) - h.method('ndim', returns='size_t', const=True) - h.method('equal', - api.params(x='const migraphx::shape&'), - invoke='migraphx::equal($@)', - returns='bool', - const=True) - h.method('standard', returns='bool', const=True) - h.method('dynamic', returns='bool', const=True) - h.method('index', api.params(i='size_t'), returns='size_t', const=True) - - -@auto_handle() -def argument(h): - h.constructor('create', - api.params(shape='const migraphx::shape&', buffer='void*')) - h.constructor('create_empty', api.params(shape='const migraphx::shape&')) - h.method('shape', - fname='get_shape', - cpp_name='get_shape', - returns='const migraphx::shape&', - const=True) - h.method('buffer', - fname='data', - cpp_name='data', - returns='char*', - const=True) - h.method('equal', - api.params(x='const migraphx::argument&'), - invoke='migraphx::equal($@)', - returns='bool', - const=True) - - -api.add_function('migraphx_argument_generate', - api.params(s='const migraphx::shape&', seed='size_t'), - fname='migraphx::generate_argument', - returns='migraphx::argument') - - -@auto_handle() -def target(h): - h.constructor('create', - api.params(name='const char*'), - fname='migraphx::get_target') - - -@api.handle('migraphx_program_parameter_shapes', - 'std::unordered_map') -def program_parameter_shapes(h): - h.method('size', returns='size_t') - h.method('get', - api.params(name='const char*'), - fname='at', - cpp_name='operator[]', - returns='const migraphx::shape&') - h.method('names', - invoke='migraphx::get_names(${program_parameter_shapes})', - returns='std::vector') - - -@api.handle('migraphx_program_parameters', - 'std::unordered_map') -def program_parameters(h): - h.constructor('create') - h.method('add', - api.params(name='const char*', - argument='const migraphx::argument&'), - invoke='${program_parameters}[${name}] = ${argument}') - - -@api.handle('migraphx_arguments', 'std::vector') -def arguments(h): - h.method('size', returns='size_t') - h.method('get', - api.params(idx='size_t'), - fname='at', - cpp_name='operator[]', - returns='const migraphx::argument&') - - -@api.handle('migraphx_shapes', 'std::vector') -def shapes(h): - h.method('size', returns='size_t') - h.method('get', - api.params(idx='size_t'), - fname='at', - cpp_name='operator[]', - returns='const migraphx::shape&') - - -@api.handle('migraphx_instruction', 'migraphx::instruction_ref') -def instruction(h): - pass - - -@api.handle('migraphx_instructions', 'std::vector') -def instructions(h): - h.constructor( - 'create', - api.params(ptr='const const_migraphx_instruction_t*', size='size_t'), - fname='migraphx::to_obj_vector') - - -@api.handle('migraphx_modules', 'std::vector') -def modules(h): - h.constructor('create', - api.params(ptr='migraphx_module_t*', size='size_t'), - fname='migraphx::to_objptr_vector') - - -@auto_handle(ref=True) -def module(h): - h.constructor('create', api.params(name='std::string')) - h.method('print', invoke='migraphx::print_module($@)', const=True) - h.method('add_instruction', - api.params(op='migraphx::operation', - args='std::vector'), - returns='migraphx::instruction_ref') - h.method('add_instruction_with_mod_args', - api.params(op='migraphx::operation', - args='std::vector', - module_refs='std::vector'), - fname='add_instruction', - returns='migraphx::instruction_ref') - h.method('add_literal', - api.params(shape='const migraphx::shape&', buffer='const char*'), - returns='migraphx::instruction_ref') - h.method('add_parameter', - api.params(name='const char*', shape='const migraphx::shape&'), - returns='migraphx::instruction_ref') - h.method('add_return', - api.params(args='std::vector'), - returns='migraphx::instruction_ref') - h.method('add_allocation', - api.params(s='const migraphx::shape&'), - invoke='migraphx::add_allocation($@)', - returns='migraphx::instruction_ref') - - -@auto_handle() -def program(h): - h.constructor('create') - h.method('get_main_module', returns='migraphx::module*') - h.method('create_module', - api.params(name='const char*'), - returns='migraphx::module*') - h.method( - 'compile', - api.params(target='migraphx::target', - options='migraphx::compile_options')) - h.method('get_parameter_shapes', - returns='std::unordered_map') - h.method('get_output_shapes', - invoke='migraphx::get_output_shapes($@)', - returns='std::vector') - h.method('print', invoke='migraphx::print_program($@)', const=True) - h.method('sort') - h.method('run', - api.params( - params='std::unordered_map'), - invoke='migraphx::run($@)', - returns='std::vector') - h.method('run_async', - api.params( - params='std::unordered_map', - s='void*', - name='const char *'), - invoke='migraphx::run_async($@)', - returns='std::vector') - h.method('equal', - api.params(x='const migraphx::program&'), - invoke='migraphx::equal($@)', - returns='bool', - const=True) - h.method('experimental_get_context', - invoke='migraphx::get_context($@)', - const=True, - returns='migraphx::context') - - -@auto_handle() -def operation(h): - h.constructor('create', - api.params(name='const char*', - attributes='const char*', - vlist='...'), - fname='migraphx::create_op') - h.method('name', returns='std::string') - - -api.add_function('migraphx_load', - api.params(name='const char*', - options='migraphx::file_options'), - fname='migraphx::load', - returns='migraphx::program') - -api.add_function('migraphx_save', - api.params(p='migraphx::program&', - name='const char*', - options='migraphx::file_options'), - fname='migraphx::save') - - -@auto_handle() -def onnx_options(h): - h.constructor('create') - h.method( - 'set_input_parameter_shape', - api.params(name='const char*', dims='std::vector'), - invoke='migraphx::set_input_parameter_shape($@)', - ) - h.method( - 'set_dyn_input_parameter_shape', - api.params(name='const char*', - dims='std::vector'), - invoke='migraphx::set_dyn_input_parameter_shape($@)', - ) - h.method( - 'set_default_dim_value', - api.params(value='size_t'), - invoke='migraphx::set_default_dim_value($@)', - ) - h.method( - 'set_default_dyn_dim_value', - api.params(dd='const migraphx::shape::dynamic_dimension&'), - invoke='migraphx::set_default_dyn_dim_value($@)', - ) - h.method( - 'set_default_loop_iterations', - api.params(value='int64_t'), - invoke='migraphx::set_default_loop_iterations($@)', - ) - h.method( - 'set_limit_loop_iterations', - api.params(value='int64_t'), - invoke='migraphx::set_limit_loop_iterations($@)', - ) - h.method( - 'set_external_data_path', - api.params(external_data_path='const char*'), - invoke='migraphx::set_external_data_path($@)', - ) - - -@auto_handle() -def file_options(h): - h.constructor('create') - h.method('set_file_format', - api.params(format='const char*'), - invoke='migraphx::set_file_format($@)') - - -@auto_handle() -def compile_options(h): - h.constructor('create') - h.method('set_offload_copy', - api.params(value='bool'), - invoke='migraphx::set_offload_copy($@)') - h.method('set_fast_math', - api.params(value='bool'), - invoke='migraphx::set_fast_math($@)') - h.method('set_exhaustive_tune_flag', - api.params(value='bool'), - invoke='migraphx::set_exhaustive_tune_flag($@)') - - -api.add_function('migraphx_parse_onnx', - api.params(name='const char*', - options='migraphx::onnx_options'), - fname='migraphx::parse_onnx', - returns='migraphx::program') - -api.add_function('migraphx_parse_onnx_buffer', - api.params(data='const void*', - size='size_t', - options='migraphx::onnx_options'), - fname='migraphx::parse_onnx_buffer', - returns='migraphx::program') - - -@auto_handle() -def tf_options(h): - h.constructor('create') - h.method( - 'set_nhwc', - api.params(is_nhwc='bool'), - invoke='migraphx::set_nhwc($@)', - ) - h.method( - 'set_input_parameter_shape', - api.params(name='const char*', dims='std::vector'), - invoke='migraphx::set_input_parameter_shape($@)', - ) - h.method( - 'set_default_dim_value', - api.params(value='size_t'), - invoke='migraphx::set_default_dim_value($@)', - ) - h.method( - 'set_output_names', - api.params(names='std::vector'), - invoke='migraphx::set_output_names($@)', - ) - - -api.add_function('migraphx_parse_tf', - api.params(name='const char*', - options='migraphx::tf_options'), - fname='migraphx::parse_tf', - returns='migraphx::program') - - -@api.handle('migraphx_quantize_op_names', 'std::vector') -def quantize_op_names(h): - h.constructor('create') - h.method('add', api.params(name='const char*'), fname='push_back') - - -api.add_function('migraphx_quantize_fp16_with_op_names', - api.params(prog='migraphx::program&', - name='std::vector&'), - fname='migraphx::quantize_fp16_with_op_names') - -api.add_function('migraphx_quantize_fp16', - api.params(prog='migraphx::program&'), - fname='migraphx::quantize_fp16') - - -@auto_handle() -def quantize_int8_options(h): - h.constructor('create') - h.method( - 'add_op_name', - api.params(name='const char*'), - invoke='migraphx::add_op_name($@)', - ) - h.method( - 'add_calibration_data', - api.params(data='std::unordered_map'), - invoke='migraphx::add_calibration_data($@)', - ) - - -api.add_function('migraphx_quantize_int8', - api.params(prog='migraphx::program&', - target='migraphx::target', - options='migraphx::quantize_int8_options'), - fname='migraphx::quantize_int8_wrap') - - -@auto_handle() -def quantize_fp8_options(h): - h.constructor('create') - h.method( - 'add_calibration_data', - api.params(data='std::unordered_map'), - invoke='migraphx::add_calibration_data($@)', - ) - - -api.add_function('migraphx_quantize_fp8', - api.params(prog='migraphx::program&', - target='migraphx::target', - options='migraphx::quantize_fp8_options'), - fname='migraphx::quantize_fp8_wrap') - - -@auto_handle(ref=True) -def context(h): - h.method('finish', const=True) - h.method('get_queue', returns='void*', fname='get_queue().unsafe_get') - - -@api.interface('migraphx_experimental_custom_op', - 'migraphx::experimental_custom_op') -def experimental_custom_op(h): - h.constructor('create', - api.params(obj_typename='const char*', name='const char*')) - h.virtual('compute', - api.params(ctx='migraphx::context', - output='migraphx::shape', - inputs='std::vector'), - returns='migraphx::argument') - h.virtual('compute_shape', - api.params(inputs='std::vector'), - returns='migraphx::shape') - h.virtual('output_alias', - api.params(inputs='std::vector'), - returns='std::vector') - h.virtual('runs_on_offload_target', returns='bool') - h.method('register', invoke='migraphx::register_custom_op($@)') diff --git a/docker/rocm/migraphx/apply_alpha_beta.cpp b/docker/rocm/migraphx/apply_alpha_beta.cpp deleted file mode 100644 index 5f86a1dc1..000000000 --- a/docker/rocm/migraphx/apply_alpha_beta.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -instruction_ref insert_apply_alpha_beta(module& m, - instruction_ref pos, - const std::vector& args, - const operation& op, - const literal& alpha, - const literal& beta) -{ - auto a = args[0]; - auto b = args[1]; - auto input_type = a->get_shape().type(); - if(not float_equal(alpha.at(0), 1.0)) - { - auto alpha_literal = m.add_literal(alpha); - a = insert_common_op(m, pos, migraphx::make_op("mul"), {alpha_literal, a}); - if(a->get_shape().type() != input_type) - { - a = m.insert_instruction(pos, make_op("convert", {{"target_type", input_type}}), a); - } - } - auto op_res = m.insert_instruction(pos, op, a, b); - if(args.size() == 3) - { - if(not float_equal(beta.at(0), 0.0) and args[2]->get_shape().elements() > 0) - { - auto out_lens = op_res->get_shape().lens(); - auto c = args[2]; - auto c_lens = c->get_shape().lens(); - input_type = c->get_shape().type(); - if(out_lens != c_lens) - { - c = m.insert_instruction( - pos, migraphx::make_op("multibroadcast", {{"out_lens", out_lens}}), args[2]); - } - auto beta_literal = m.add_literal(beta); - auto beta_c = insert_common_op(m, pos, migraphx::make_op("mul"), {c, beta_literal}); - if(beta_c->get_shape().type() != input_type) - { - beta_c = m.insert_instruction( - pos, migraphx::make_op("convert", {{"target_type", input_type}}), beta_c); - } - return m.insert_instruction(pos, migraphx::make_op("add"), op_res, beta_c); - } - } - return op_res; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/argument.cpp b/docker/rocm/migraphx/argument.cpp deleted file mode 100644 index 733e27aca..000000000 --- a/docker/rocm/migraphx/argument.cpp +++ /dev/null @@ -1,210 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -argument::argument(const shape& s) : m_shape(s) -{ - auto buffer = make_shared_array(s.bytes()); - assign_buffer({[=]() mutable { return buffer.get(); }}); -} - -argument::argument(shape s, std::nullptr_t) - : m_shape(std::move(s)), m_data({[] { return nullptr; }}) -{ -} - -argument::argument(const shape& s, const argument::data_t& d) : m_shape(s), m_data(d) {} - -void argument::assign_buffer(std::function d) -{ - const shape& s = m_shape; - if(s.type() != shape::tuple_type) - { - m_data = {std::move(d)}; - return; - } - // Collect all shapes - std::unordered_map shapes; - { - std::size_t i = 0; - fix([&](auto self, auto ss) { - if(ss.sub_shapes().empty()) - { - shapes[i] = ss; - i++; - } - else - { - for(auto&& child : ss.sub_shapes()) - self(child); - } - })(s); - } - // Sort by type size - std::vector order(shapes.size()); - std::iota(order.begin(), order.end(), 0); - std::sort(order.begin(), order.end(), by(std::greater<>{}, [&](auto i) { - return shapes[i].type_size(); - })); - // Compute offsets - std::unordered_map offsets; - std::size_t offset = 0; - for(auto i : order) - { - offsets[i] = offset; - offset += shapes[i].bytes(); - } - assert(offset == s.bytes()); - - std::size_t i = 0; - m_data = fix([&](auto self, auto ss) { - data_t result; - if(ss.sub_shapes().empty()) - { - auto n = offsets[i]; - result = {[d, n]() mutable { return d() + n; }}; - i++; - return result; - } - std::vector subs; - std::transform(ss.sub_shapes().begin(), - ss.sub_shapes().end(), - std::back_inserter(subs), - [&](auto child) { return self(child); }); - result.sub = subs; - return result; - })(s); -} - -std::vector flatten(const std::vector& args) -{ - std::vector result; - for(const auto& arg : args) - { - if(arg.get_shape().type() == shape::tuple_type) - { - auto subs = flatten(arg.get_sub_objects()); - result.insert(result.end(), subs.begin(), subs.end()); - } - else - { - result.push_back(arg); - } - } - return result; -} - -std::vector to_shapes(const std::vector& args) -{ - std::vector shapes; - std::transform(args.begin(), args.end(), std::back_inserter(shapes), [](auto&& arg) { - return arg.get_shape(); - }); - return shapes; -} - -argument::argument(const std::vector& args) - : m_shape(to_shapes(args)), m_data(data_t::from_args(args)) -{ -} - -char* argument::data() const -{ - assert(m_shape.type() != shape::tuple_type); - assert(not this->empty()); - return m_data.get(); -} - -bool argument::empty() const { return not m_data.get and m_data.sub.empty(); } - -const shape& argument::get_shape() const { return this->m_shape; } - -argument argument::reshape(const shape& s) const -{ - assert(s.element_space() <= this->get_shape().element_space()); - return {s, this->m_data}; -} - -argument::data_t argument::data_t::share() const -{ - data_t result; - if(this->get) - { - auto self = std::make_shared(*this); - result.get = [self]() mutable { return self->get(); }; - } - std::transform(sub.begin(), sub.end(), std::back_inserter(result.sub), [](const auto& d) { - return d.share(); - }); - return result; -} - -argument::data_t argument::data_t::from_args(const std::vector& args) -{ - data_t result; - std::transform(args.begin(), args.end(), std::back_inserter(result.sub), [](auto&& arg) { - return arg.m_data; - }); - return result; -} - -argument argument::copy() const -{ - argument result{this->get_shape()}; - auto* src = this->data(); - std::copy(src, src + this->get_shape().bytes(), result.data()); - return result; -} - -argument argument::share() const { return {m_shape, m_data.share()}; } - -std::vector argument::get_sub_objects() const -{ - std::vector result; - assert(m_shape.sub_shapes().size() == m_data.sub.size()); - std::transform(m_shape.sub_shapes().begin(), - m_shape.sub_shapes().end(), - m_data.sub.begin(), - std::back_inserter(result), - [](auto&& s, auto&& d) { - return argument{s, d}; - }); - return result; -} - -argument argument::element(std::size_t i) const -{ - assert(this->get_shape().sub_shapes().empty()); - auto idx = this->get_shape().index(i); - auto offset = this->get_shape().type_size() * idx; - return argument{shape{this->get_shape().type()}, this->data() + offset}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/auto_contiguous.cpp b/docker/rocm/migraphx/auto_contiguous.cpp deleted file mode 100644 index d08331dc2..000000000 --- a/docker/rocm/migraphx/auto_contiguous.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void auto_contiguous::apply(module& m) const -{ - std::string key = "require_std_shape"; - for(auto ins : reverse_iterator_for(m)) - { - auto&& attr = ins->get_operator().attributes(); - if((attr.get(key, false))) - { - auto args = ins->inputs(); - auto new_args = args; - std::transform(args.begin(), args.end(), new_args.begin(), [&](auto in) { - if(in->name() == "contiguous") - { - return in; - } - return m.insert_instruction(ins, make_op("contiguous"), in); - }); - - if(new_args != args) - { - m.replace_instruction(ins, ins->get_operator(), new_args); - } - } - } - - auto last = std::prev(m.end()); - for(auto ins : iterator_for(m)) - { - if(contains({"layout", "@return"}, ins->name())) - continue; - // for last instruction that is NOT a return - if(ins->outputs().empty() and ins != last) - continue; - shape s = ins->get_shape(); - // If s is not standard layout or has out of sequence strides, insert "contiguous" op - // to make a standard shape - if(not s.dynamic() and (not s.standard() or s.normalize_standard() != s) and - s.elements() > 1) - { - auto c = m.insert_instruction(std::next(ins), make_op("contiguous"), ins); - m.replace_instruction(ins, c); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/autocast_fp8.cpp b/docker/rocm/migraphx/autocast_fp8.cpp deleted file mode 100644 index d4ba4b56a..000000000 --- a/docker/rocm/migraphx/autocast_fp8.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void autocast_fp8_pass::apply(module& m) const -{ - std::vector remove_parameters; - for(auto ins : iterator_for(m)) - { - const auto& ins_name = ins->name(); - if(ins_name == "@param" and contains(fp8_types{}.get(), ins->get_shape().type())) - { - shape::type_t fp8_type = ins->get_shape().type(); - migraphx::shape new_shape = ins->get_shape().with_type(target_type); - std::string param_name = ins->get_operator().to_value()["parameter"].to(); - m.rename_parameter(ins, param_name + "_old"); - auto new_param = m.add_parameter(param_name, new_shape); - auto new_ins = m.insert_instruction( - ins, - migraphx::make_op("convert", {{"target_type", migraphx::to_value(fp8_type)}}), - new_param); - m.replace_instruction(ins, new_ins); - remove_parameters.push_back(ins); - } - - if(ins_name == "@return") - { - std::vector inputs = ins->inputs(); - std::vector new_inputs; - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(new_inputs), [&](auto i) { - if(contains(fp8_types{}.get(), i->get_shape().type())) - { - return m.insert_instruction( - ins, - migraphx::make_op("convert", - {{"target_type", migraphx::to_value(target_type)}}), - i); - } - else - return i; - }); - m.replace_return({new_inputs}); - } - } - // Remove unused parameters with fp8 type - for(const auto& i : remove_parameters) - m.remove_instruction(i); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/base64.cpp b/docker/rocm/migraphx/base64.cpp deleted file mode 100644 index 0d08b1f62..000000000 --- a/docker/rocm/migraphx/base64.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace { -using byte = unsigned char; - -std::array constexpr b64_chars{ - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - -/// base64 encoder snippet altered from https://stackoverflow.com/a/37109258 -std::string encode(const std::vector& buf) -{ - std::size_t len = buf.size(); - std::vector res_vec((len + 2) / 3 * 4, '='); - std::size_t j = 0; - std::size_t remaining = len % 3; - const size_t last = len - remaining; - - for(size_t i = 0; i < last; i += 3) - { - std::size_t n = static_cast(buf.at(i)) << 16u | - static_cast(buf.at(i + 1)) << 8u | - static_cast(buf.at(i + 2)); - res_vec.at(j++) = b64_chars.at(n >> 18u); - res_vec.at(j++) = b64_chars.at(n >> 12u & 0x3Fu); - res_vec.at(j++) = b64_chars.at(n >> 6u & 0x3Fu); - res_vec.at(j++) = b64_chars.at(n & 0x3Fu); - } - // Set padding - if(remaining != 0) - { - std::size_t n = --remaining == 0 ? static_cast(buf.at(last)) - : static_cast(buf.at(last)) << 8u | - static_cast(buf.at(last + 1)); - res_vec.at(j++) = b64_chars.at(remaining == 0 ? n >> 2u : n >> 10u & 0x3Fu); - res_vec.at(j++) = b64_chars.at(remaining == 0 ? n << 4u & 0x3Fu : n >> 4u & 0x03Fu); - res_vec.at(j++) = remaining == 0 ? '=' : b64_chars.at(n << 2u & 0x3Fu); - } - return {res_vec.begin(), res_vec.end()}; -} - -} // namespace - -std::string base64_encode(const std::string& str) -{ - return encode(std::vector(str.begin(), str.end())); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/common.cpp b/docker/rocm/migraphx/common.cpp deleted file mode 100644 index 83fca7c82..000000000 --- a/docker/rocm/migraphx/common.cpp +++ /dev/null @@ -1,254 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -std::vector compute_broadcasted_lens(std::vector s0, - std::vector s1) -{ - if(s0 == s1) - return s0; - if(s0.size() > s1.size()) - s0.swap(s1); - std::vector out_lens(s1); - auto offset = s1.size() - s0.size(); - std::transform( - s0.begin(), s0.end(), s1.begin() + offset, out_lens.begin() + offset, [&](auto a, auto b) { - if(a != b and a != 1 and b != 1) - { - MIGRAPHX_THROW("COMPUTE_BROADCASTLEN: shape {" + migraphx::to_string_range(s0) + - "} and {" + migraphx::to_string_range(s1) + "} mismatch!"); - } - return std::max(a, b); - }); - return out_lens; -} -std::vector -compute_broadcasted_dyn_dims(std::vector dds0, - std::vector dds1) -{ - if(dds0.size() > dds1.size()) - { - std::swap(dds0, dds1); - } - auto offset = dds1.size() - dds0.size(); - std::vector out_dims(dds1); - std::transform(dds0.cbegin(), - dds0.cend(), - dds1.cbegin() + offset, - out_dims.begin() + offset, - [&](auto a, auto b) { - if(a == b or b == 1) - { - return a; - } - else if(a == 1) - { - return b; - } - else - { - auto intersect = a.intersection(b); - if(intersect.has_value()) - { - return intersect.value(); - } - MIGRAPHX_THROW("COMPUTE_BROADCASTED_DYN_DIMS: dynamic shapes {" + - migraphx::to_string_range(dds0) + "} and {" + - migraphx::to_string_range(dds1) + "} mismatch!"); - } - }); - return out_dims; -} - -std::vector compute_broadcasted_dyn_dims(shape s0, shape s1) -{ - // change both shapes to dynamic_dimension representation - s0 = s0.to_dynamic(); - s1 = s1.to_dynamic(); - return compute_broadcasted_dyn_dims(s0.dyn_dims(), s1.dyn_dims()); -} - -std::vector compute_common_dyn_dims(const std::vector& shapes) -{ - auto ret_shape = shapes.at(0); - std::for_each(shapes.cbegin() + 1, shapes.cend(), [&](auto s) { - ret_shape = shape{ret_shape.type(), compute_broadcasted_dyn_dims(ret_shape, s)}; - }); - return ret_shape.dyn_dims(); -} - -std::vector compute_common_lens(const std::vector& shapes) -{ - assert(not shapes.empty()); - assert( - std::none_of(shapes.cbegin(), shapes.cend(), [](auto shape) { return shape.dynamic(); })); - return transform_accumulate(shapes.begin() + 1, - shapes.end(), - shapes.front().lens(), - &compute_broadcasted_lens, - [](auto s) { return s.lens(); }); -} - -shape::type_t compute_common_type(shape::type_t t1, shape::type_t t2) -{ - if(t1 == t2) - return t1; - shape::type_t result; - shape::visit(t1, [&](auto x) { - shape::visit(t2, [&](auto y) { - // Workaround broken warning on gcc 5 - (void)x; - (void)y; - using type = std::common_type_t; - result = shape::get_type{}; - }); - }); - return result; -} - -shape::type_t compute_common_types(const std::vector& shapes) -{ - assert(not shapes.empty()); - return transform_accumulate( - shapes.begin() + 1, shapes.end(), shapes.front().type(), &compute_common_type, [&](auto s) { - return s.type(); - }); -} - -shape common_shape(const std::vector& shapes) -{ - if(shapes.empty()) - return {}; - return {compute_common_types(shapes), compute_common_lens(shapes)}; -} - -std::vector insert_common_args(module& m, - instruction_ref ins, - std::vector inputs, - common_options options) -{ - if(std::any_of( - inputs.cbegin(), inputs.cend(), [](auto input) { return input->get_shape().dynamic(); })) - { - auto input_shapes = to_shapes(inputs); - if(options.common_lens) - { - auto c_dyn_dims = compute_common_dyn_dims(input_shapes); - - auto s0 = inputs[0]->get_shape(); - // always add both multibroadcast instructions for dynamic shapes - inputs[0] = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_dyn_dims", to_value(c_dyn_dims)}}), inputs); - std::transform(inputs.begin() + 1, inputs.end(), inputs.begin() + 1, [&](auto input) { - // uses previous input to avoid recalculating the common shape from the - // full set of input shapes at runtime - auto s = input->get_shape(); - return m.insert_instruction( - ins, - make_op("multibroadcast", {{"out_dyn_dims", to_value(c_dyn_dims)}}), - input, - inputs[0]); - }); - } - if(options.common_type) - { - auto c_type = compute_common_types(input_shapes); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](auto input) { - if(input->get_shape().type() != c_type) - { - input = m.insert_instruction( - ins, make_op("convert", {{"target_type", c_type}}), input); - } - return input; - }); - } - } - else - { - auto common = common_shape(to_shapes(inputs)); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](auto input) { - if(options.common_lens and input->get_shape().lens() != common.lens()) - { - input = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", common.lens()}}), input); - } - if(options.common_type and input->get_shape().type() != common.type()) - { - input = m.insert_instruction( - ins, make_op("convert", {{"target_type", common.type()}}), input); - } - return input; - }); - } - return inputs; -} - -std::vector -add_common_args(module& m, std::vector inputs, common_options options) -{ - return insert_common_args(m, m.end(), std::move(inputs), options); -} - -instruction_ref insert_common_op(module& m, - instruction_ref ins, - const operation& op, - std::vector inputs, - common_options options) -{ - return m.insert_instruction(ins, op, insert_common_args(m, ins, std::move(inputs), options)); -} - -instruction_ref add_common_op(module& m, - const operation& op, - std::vector inputs, - common_options options) -{ - return insert_common_op(m, m.end(), op, std::move(inputs), options); -} - -shape make_bcast_shape(const shape& input_shape, const std::vector& bcast_lens) -{ - assert(not input_shape.dynamic()); - auto offset = bcast_lens.size() - input_shape.ndim(); - std::vector bcast_strides(bcast_lens.size(), 0); - for(std::ptrdiff_t i : reverse(range(input_shape.ndim()))) - { - if(bcast_lens.at(i + offset) == input_shape.lens()[i]) - { - bcast_strides.at(i + offset) = input_shape.strides()[i]; - } - } - return shape{input_shape.type(), bcast_lens, bcast_strides}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/common_dims.cpp b/docker/rocm/migraphx/common_dims.cpp deleted file mode 100644 index 1afe92087..000000000 --- a/docker/rocm/migraphx/common_dims.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -static auto compute_end_dim(Iterator start, Iterator last, std::size_t dim) -{ - std::size_t x = 1; - auto it = std::find_if(start, last, [&](auto i) { - x *= i; - return x > dim; - }); - if(x < dim) - return start; - return it; -} - -struct common_dim_state -{ - common_dim_state(const std::vector& pdims, - std::vector>& paxes_map) - : dims(&pdims), axes_map(&paxes_map), it(dims->begin()) - { - } - const std::vector* dims = nullptr; - std::vector>* axes_map = nullptr; - std::vector::const_iterator it{}; - std::size_t rem = 1; - std::size_t get() const { return *it / rem; } - bool is_end() const { return it == dims->end(); } - void next(std::size_t i = 1) { it += i; } - auto dims_for(std::size_t d) const - { - auto dim_end = compute_end_dim(it, dims->end(), d); - return range(it, dim_end); - } - void add_axes(std::size_t naxes, std::size_t start) MIGRAPHX_TIDY_CONST - { - auto axes = compute_axes(naxes, start); - axes_map->push_back(std::move(axes)); - } - - void add_multi_axes(std::size_t naxes, std::size_t start) MIGRAPHX_TIDY_CONST - { - auto axes = compute_axes(naxes, start); - std::transform(axes.begin(), - axes.end(), - std::back_inserter(*axes_map), - [&](auto axis) -> std::vector { return {axis}; }); - } - std::vector compute_axes(std::size_t naxes, std::size_t start) const - { - if(rem != 1) - { - assert(start > 0); - naxes++; - start--; - } - std::vector axes(naxes); - std::iota(axes.begin(), axes.end(), start); - return axes; - } -}; - -static bool compute_common_dim(std::vector& cd_dims, - common_dim_state& state1, - common_dim_state& state2) -{ - assert(state1.get() < state2.get()); - auto d2 = state2.get(); - auto dims = state1.dims_for(d2); - auto n = elements(dims); - auto naxes = distance(dims); - if(naxes == 0) - return false; - // If not divisible then we can't compute a common dim - if((d2 % n) != 0) - return false; - auto rem = d2 / n; - state1.add_multi_axes(naxes, cd_dims.size()); - state2.add_axes(rem == 1 ? naxes : naxes + 1, cd_dims.size()); - - state1.rem = rem; - state2.rem = 1; - - cd_dims.insert(cd_dims.end(), dims.begin(), dims.end()); - if(state1.rem != 1) - cd_dims.push_back(state1.rem); - state1.next(distance(dims)); - state2.next(); - return true; -} - -common_dims common_dims::compute(const std::vector& dims1, - const std::vector& dims2) -{ - assert(elements(dims1) > 0); - assert(elements(dims1) == elements(dims2)); - common_dims cd; - common_dim_state state1{dims1, cd.axes_map1}; - common_dim_state state2{dims2, cd.axes_map2}; - while(not state1.is_end() and not state2.is_end()) - { - auto d1 = state1.get(); - auto d2 = state2.get(); - if(d1 == d2) - { - state1.add_axes(1, cd.dims.size()); - state2.add_axes(1, cd.dims.size()); - state1.rem = 1; - state2.rem = 1; - cd.dims.push_back(d1); - state1.next(); - state2.next(); - } - else if(d1 < d2) - { - if(not compute_common_dim(cd.dims, state1, state2)) - return {}; - } - else // if(d1 > d2) - { - if(not compute_common_dim(cd.dims, state2, state1)) - return {}; - } - } - assert(elements(dims1) == elements(cd.dims)); - return cd; -} - -const std::vector>* common_dims::get_axes_map(std::size_t n) const -{ - if(axes_map1.size() == n) - return &axes_map1; - if(axes_map2.size() == n) - return &axes_map2; - return nullptr; -} - -std::vector -common_dims::get_dimensions_for(const std::vector& idims) const -{ - if(dims.size() == idims.size()) - return idims; - if(elements(dims) == elements(idims)) - return dims; - // Bail for now since its ambiguous which axes map can be used - // TODO: Check for similiarity - if(axes_map1.size() == axes_map2.size()) - return {}; - const auto* axes_map = get_axes_map(idims.size()); - if(axes_map == nullptr) - return {}; - auto xdims = dims; - for(auto i : range(axes_map->size())) - { - auto dim = idims[i]; - const auto& axes = (*axes_map)[i]; - if(axes.size() == 1) - { - xdims[axes.front()] = dim; - } - else if(dim == 1) - { - for(auto axis : axes) - xdims[axis] = 1; - } - } - if(elements(xdims) == elements(idims)) - return xdims; - return {}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/compile_src.cpp b/docker/rocm/migraphx/compile_src.cpp deleted file mode 100644 index e3ebd924b..000000000 --- a/docker/rocm/migraphx/compile_src.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -std::vector src_compiler::compile(const std::vector& srcs) const -{ - assert(not srcs.empty()); - tmp_dir td{"compile"}; - std::vector params{flags}; - - params.emplace_back("-I."); - - auto out = output; - - for(const auto& src : srcs) - { - fs::path full_path = td.path / src.path; - fs::path parent_path = full_path.parent_path(); - fs::create_directories(parent_path); - write_buffer(full_path, src.content.data(), src.content.size()); - if(src.path.extension().string() == ".cpp") - { - params.emplace_back(src.path.filename().string()); - if(out.empty()) - out = src.path.stem().string() + out_ext; - } - } - - params.emplace_back("-o " + out); - - std::vector args; - if(not launcher.empty()) - args.push_back(compiler.string()); - args.insert(args.end(), params.begin(), params.end()); - td.execute(launcher.empty() ? compiler : launcher, args); - - auto out_path = td.path / out; - if(not fs::exists(out_path)) - MIGRAPHX_THROW("Output file missing: " + out); - - return read_buffer(out_path); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/convert_to_json.cpp b/docker/rocm/migraphx/convert_to_json.cpp deleted file mode 100644 index 0dd823d1e..000000000 --- a/docker/rocm/migraphx/convert_to_json.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -std::vector json_tokenize(const std::string& s) -{ - std::vector lexers; - - // Quote - lexers.push_back([](const char* start, const char* end) { - if(*start != '\"') - return start; - ++start; - while((start != end) and (*start != '\"')) - { - if(*start == '\\') - start++; - start++; - } - - return ++start; - }); - - // Line comments - lexers.push_back([](const char* start, const char* end) { - if(*start == '#') - start++; - else if((start + 1) < end and start[0] == '/' and start[1] == '/') - start += 2; - else - return start; - return std::find_if(start, end, [&](char c) { return c == '\n'; }); - }); - - // Whitespace - lexers.push_back(lex_while(&isspace)); - - // Punctation - lexers.push_back(lex_if(&ispunct)); - - // Identifier/number - lexers.push_back(lex_while([](char c) { - return (isalnum(c) != 0 or contains({'_', '.', '+'}, c)); - })); - - return tokenize(s.data(), s.data() + s.length(), lexers); -} - -std::string convert_to_json(const std::string& str) -{ - auto tokens = json_tokenize(str); - std::stringstream ss; - - for(auto& token : tokens) - { - std::string s(token); - if(starts_with(s, "#") or starts_with(s, "//")) - continue; - if(std::isalpha(s.front()) != 0 and - not contains({"null", "nan", "true", "false", "inf"}, s)) - { - ss << "\"" << s << "\""; - } - else - { - ss << s; - } - } - - return ss.str(); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/cpp_generator.cpp b/docker/rocm/migraphx/cpp_generator.cpp deleted file mode 100644 index 2aa808778..000000000 --- a/docker/rocm/migraphx/cpp_generator.cpp +++ /dev/null @@ -1,284 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -cpp_generator::function& -cpp_generator::function::set_body(const module& m, const cpp_generator::generate_module_callback& g) -{ - const std::string prefix = "zz"; - std::unordered_map names; - std::stringstream ss; - - auto return_ins = std::prev(m.end()); - - for(auto ins : iterator_for(m)) - { - ss << "// " << ins->get_operator() << " -> " << ins->get_shape() << "\n"; - if(ins->name() == "@param") - { - names[ins] = to_c_id( - migraphx::any_cast(ins->get_operator()).parameter); - } - else if(ins->name() == "@return") - { - names[ins] = prefix + "return"; - ss << "auto " << names[ins] << " = " << g(ins, names) << ";\n"; - return_ins = ins; - } - else - { - std::string n = prefix + std::to_string(names.size()); - names[ins] = n; - ss << "auto " << n << " = " << g(ins, names) << ";\n"; - } - } - ss << "return " << names.at(return_ins) << ";\n"; - body = ss.str(); - return *this; -} - -cpp_generator::function& cpp_generator::function::set_types(const module& m) -{ - return cpp_generator::function::set_types(m, [](auto s) { return shape::cpp_type(s.type()); }); -} -cpp_generator::function& -cpp_generator::function::set_types(const module& m, const std::function& parse) -{ - this->params.clear(); - auto pmap = m.get_parameter_shapes(); - std::map input_map(pmap.begin(), pmap.end()); - std::transform( - input_map.begin(), input_map.end(), std::back_inserter(this->params), [&](auto&& p) { - return param{p.first, parse(p.second)}; - }); - auto output_shapes = m.get_output_shapes(); - assert(not output_shapes.empty()); - this->return_type = parse(output_shapes.front()); - return *this; -} - -cpp_generator::function& cpp_generator::function::set_generic_types(const module& m) -{ - this->params.clear(); - auto pmap = m.get_parameter_shapes(); - std::map input_map(pmap.begin(), pmap.end()); - std::transform( - input_map.begin(), input_map.end(), std::back_inserter(this->params), [&](auto&& p) { - return param{p.first, "T" + to_c_id(p.first)}; - }); - - std::transform(input_map.begin(), - input_map.end(), - std::back_inserter(this->tparams), - [&](auto&& p) { return "class T" + to_c_id(p.first); }); - this->return_type = "auto"; - return *this; -} - -cpp_generator::function& cpp_generator::function::unused_param(const std::string& pname) -{ - body.insert(0, "(void)" + pname + ";\n"); - return *this; -} -cpp_generator::function& cpp_generator::function::add_generic_param(const std::string& pname) -{ - params.push_back({pname, "T" + pname}); - tparams.push_back("class T" + pname); - return *this; -} - -struct cpp_generator_impl -{ - std::stringstream fs{}; - std::size_t function_count = 0; - std::function fmap = nullptr; - std::function fresult = nullptr; - std::unordered_map point_op_map = {}; - bool always_return_tuple = false; -}; -cpp_generator::cpp_generator() : impl(std::make_unique()) {} - -cpp_generator::cpp_generator(cpp_generator&&) noexcept = default; - -cpp_generator& cpp_generator::operator=(cpp_generator rhs) -{ - std::swap(impl, rhs.impl); - return *this; -} - -cpp_generator::~cpp_generator() noexcept = default; - -void cpp_generator::fmap(const std::function& f) { impl->fmap = f; } - -void cpp_generator::fresult(const std::function& f) { impl->fresult = f; } - -void cpp_generator::always_return_tuple(bool b) { impl->always_return_tuple = b; } - -void cpp_generator::add_point_op(const std::string& op_name, const std::string& code) -{ - impl->point_op_map[op_name] = code; -} - -std::string cpp_generator::generate_point_op(const operation& op, - const std::vector& args) -{ - auto v = op.to_value(); - std::string code; - if(contains(impl->point_op_map, op.name())) - { - code = impl->point_op_map.at(op.name()); - } - else - { - auto attributes = op.attributes(); - if(not attributes.contains("point_op")) - MIGRAPHX_THROW("op is missing point_op attribute: " + op.name()); - code = attributes["point_op"].to(); - } - return interpolate_string(code, [&](auto start, auto last) -> std::string { - auto key = trim({start, last}); - if(key.empty()) - MIGRAPHX_THROW("Empty parameter"); - std::string fselector = "function:"; - if(starts_with(key, fselector)) - { - auto fname = key.substr(fselector.size()); - if(impl->fmap == nullptr) - return fname; - else - return impl->fmap(fname); - } - else if(with_char(::isdigit)(key[0])) - { - auto i = std::stoul(key); - if(i >= args.size()) - MIGRAPHX_THROW("Invalid argument index: " + key); - return args.at(i); - } - else if(v.contains(key)) - { - return v[key].template to(); - } - else - { - return key; - } - }); -} - -std::string cpp_generator::str() const { return impl->fs.str(); } - -cpp_generator::function cpp_generator::generate_module(const module& m, - const generate_module_callback& g) -{ - function f; - f.set_name(to_c_id(m.name())) - .set_types(m) - .set_body(m, [&](instruction_ref ins, const auto& names) -> std::string { - if(ins->name() == "@literal") - { - std::string string_literal; - ins->get_literal().visit([&](auto v) { - assert(v.size() == 1); - auto x = v.front(); - if(std::isinf(static_cast(x))) - { - string_literal = "__builtin_huge_val()"; - if(x < 0) - string_literal = "-__builtin_huge_val()"; - } - else if(std::isnan(static_cast(x))) - string_literal = "__builtin_nan(\"0\")"; - else - string_literal = ins->get_literal().to_string(); - }); - return shape::cpp_type(ins->get_shape().type()) + "(" + string_literal + ")"; - } - if(ins->name() == "@return") - { - // TODO: Customize the make_tuple call - if(impl->always_return_tuple or ins->inputs().size() != 1) - return "make_tuple(" + join_strings(to_args(ins->inputs(), names), ", ") + ")"; - return names.at(ins->inputs().front()); - } - auto s = g(ins, names); - if(impl->fresult) - return impl->fresult(ins->get_shape()) + '(' + s + ')'; - else - return s; - }); - return f; -} - -std::vector -cpp_generator::to_args(const std::vector& inputs, - const std::unordered_map& names) -{ - std::vector args; - std::transform(inputs.begin(), inputs.end(), std::back_inserter(args), [&](auto i) { - return names.at(i); - }); - return args; -} - -cpp_generator::function cpp_generator::generate_module(const module& m) -{ - return this->generate_module(m, [&](auto ins, const auto& names) { - return this->generate_point_op(ins->get_operator(), to_args(ins->inputs(), names)); - }); -} - -std::string cpp_generator::create_function(const cpp_generator::function& f) -{ - impl->function_count++; - if(not f.tparams.empty()) - impl->fs << "template<" << join_strings(f.tparams, ", ") << ">\n"; - std::string name = f.name.empty() ? "f" + std::to_string(impl->function_count) : f.name; - impl->fs << join_strings(f.attributes, " ") << " " << f.return_type << " " << name; - char delim = '('; - if(f.params.empty()) - impl->fs << delim; - for(auto&& p : f.params) - { - impl->fs << delim << p.type << " " << to_c_id(p.name); - delim = ','; - } - impl->fs << ") {\n" << f.body << "\n}\n"; - return name; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/dead_code_elimination.cpp b/docker/rocm/migraphx/dead_code_elimination.cpp deleted file mode 100644 index 21063836b..000000000 --- a/docker/rocm/migraphx/dead_code_elimination.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void dead_code_elimination::apply(program& p) const { p.remove_unused_modules(); } - -void dead_code_elimination::apply(module& m) const -{ - auto last = std::prev(m.end()); - for(auto ins : iterator_for(m)) - { - // Skip the first instruction, since we always process the previous - // instruction - if(ins == m.begin()) - continue; - const auto i = std::prev(ins); - // Skip the last instruction - if(i == last) - break; - // Skip instruction with empty shape as output unless its [dynamic, builtin, undefined, - // identity, allocate or tuple_type] - if((not i->get_shape().dynamic() and - (i->get_shape().elements() == 0 and - i->get_shape().type() != migraphx::shape::tuple_type)) and - not(i->name().front() == '@') and not contains({"identity", "allocate"}, i->name()) and - not i->is_undefined()) - continue; - assert(std::distance(m.begin(), i) <= std::distance(m.begin(), last)); - std::unordered_set visited; - fix([&](auto self, auto leaf) { - if(not m.has_instruction(leaf)) - return; - - if(leaf->outputs().empty()) - { - // Dont visit inputs twice - if(not visited.insert(leaf).second) - return; - std::unordered_set args(leaf->inputs().begin(), - leaf->inputs().end()); - leaf->clear_arguments(); - assert(std::distance(m.begin(), leaf) < std::distance(m.begin(), last)); - assert(leaf != ins); - if(leaf->name() != "@param") - m.move_instruction(leaf, m.end()); - for(auto arg : args) - self(arg); - } - })(i); - } - m.remove_instructions(std::next(last), m.end()); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/dom_info.cpp b/docker/rocm/migraphx/dom_info.cpp deleted file mode 100644 index 6acba63da..000000000 --- a/docker/rocm/migraphx/dom_info.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -bool dominator_info::strictly_dominate(instruction_ref ins1, instruction_ref ins2) const -{ - if(ins1 == ins2) - return false; - auto iter = ins2idom.find(ins2); - while(iter != ins2idom.end()) - { - if(ins1 == iter->second) - return true; - assert(iter != ins2idom.find(iter->second)); - iter = ins2idom.find(iter->second); - } - return false; -} - -struct module_visitor -{ - const module* mm; - const module& get_nodes() const { return *mm; } - - const std::vector& get_children(instruction_ref ins) { return ins->inputs(); } -}; - -template -dominator_info compute_dominator_generic(Visitor v) -{ - dominator_info info; - std::unordered_map> instr2_doms; - for(instruction_ref ins : iterator_for(v.get_nodes())) - { - const std::vector& children = v.get_children(ins); - if(children.size() == 1) - { - info.ins2idom[ins] = children.front(); - instr2_doms[ins] = instr2_doms[children.front()]; - } - else if(children.size() > 1) - { - auto&& doms = instr2_doms[ins]; - - doms = instr2_doms[children.front()]; - std::for_each(children.begin() + 1, children.end(), [&](instruction_ref child) { - auto&& child_doms = instr2_doms[child]; - erase_if(doms, [&](auto x) { return not contains(child_doms, x); }); - }); - auto iter = std::find_if(doms.begin(), doms.end(), [&](auto dom1) { - return std::none_of(doms.begin(), doms.end(), [&](auto dom2) { - if(dom1 == dom2) - return false; - return info.strictly_dominate(dom1, dom2); - }); - }); - if(iter != doms.end()) - info.ins2idom[ins] = *iter; - } - instr2_doms[ins].insert(ins); - } - return info; -} - -dominator_info compute_dominator(const module& m) -{ - return compute_dominator_generic(module_visitor{&m}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/CMakeLists.txt b/docker/rocm/migraphx/driver/CMakeLists.txt deleted file mode 100644 index cc096f2d3..000000000 --- a/docker/rocm/migraphx/driver/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -add_executable(driver - main.cpp - verify.cpp - passes.cpp - mlir.cpp - models.cpp - perf.cpp - marker_roctx.cpp -) -set_target_properties(driver PROPERTIES OUTPUT_NAME migraphx-driver) -if(NOT WIN32) - # Copy driver for backwards compatibility (Linux only) - add_custom_command( - TARGET driver - POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - $ - ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/driver - BYPRODUCTS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/driver - ) - set_directory_properties(PROPERTIES ADDITIONAL_CLEAN_FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/driver) -endif() -rocm_clang_tidy_check(driver) - -file(STRINGS "${CMAKE_SOURCE_DIR}/test/onnx/.onnxrt-commit" String_output) -target_compile_definitions(driver PUBLIC MIGRAPHX_ORT_SHA1="${String_output}") - -target_link_libraries(driver migraphx_all_targets migraphx_onnx migraphx_tf) - -if(MIGRAPHX_ENABLE_PYTHON) - target_link_libraries(driver migraphx_py) - target_compile_definitions(driver PRIVATE MIGRAPHX_ENABLE_PYTHON) -endif() - -rocm_install_targets( - TARGETS driver -) diff --git a/docker/rocm/migraphx/driver/argument_parser.hpp b/docker/rocm/migraphx/driver/argument_parser.hpp deleted file mode 100644 index f7f951f5c..000000000 --- a/docker/rocm/migraphx/driver/argument_parser.hpp +++ /dev/null @@ -1,748 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ARGUMENT_PARSER_HPP -#define MIGRAPHX_GUARD_RTGLIB_ARGUMENT_PARSER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#endif - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef MIGRAPHX_USE_CLANG_TIDY -#define MIGRAPHX_DRIVER_STATIC -#else -#define MIGRAPHX_DRIVER_STATIC static -#endif - -template -using bare = std::remove_cv_t>; - -namespace detail { - -template -auto is_container(int, T&& x) -> decltype(x.insert(x.end(), *x.begin()), std::true_type{}); - -template -std::false_type is_container(float, T&&); - -} // namespace detail - -template -struct is_container : decltype(detail::is_container(int(0), std::declval())) -{ -}; - -template -using is_multi_value = - std::integral_constant{} and not std::is_convertible{})>; - -enum class color -{ - reset = 0, - bold = 1, - underlined = 4, - fg_red = 31, - fg_green = 32, - fg_yellow = 33, - fg_blue = 34, - fg_default = 39, - bg_red = 41, - bg_green = 42, - bg_yellow = 43, - bg_blue = 44, - bg_default = 49 -}; -inline std::ostream& operator<<(std::ostream& os, const color& c) -{ -#ifndef _WIN32 - static const bool use_color = isatty(STDOUT_FILENO) != 0; - if(use_color) - return os << "\033[" << static_cast(c) << "m"; -#else - (void)c; -#endif - return os; -} - -inline std::string colorize(color c, const std::string& s) -{ - std::stringstream ss; - ss << c << s << color::reset; - return ss.str(); -} - -template -struct type_name -{ - static const std::string& apply() { return migraphx::get_type_name(); } -}; - -template <> -struct type_name -{ - static const std::string& apply() - { - static const std::string name = "std::string"; - return name; - } -}; - -template -struct type_name> -{ - static const std::string& apply() - { - static const std::string name = "std::vector<" + type_name::apply() + ">"; - return name; - } -}; - -template -struct value_parser -{ - template {} and not is_multi_value{})> - static T apply(const std::string& x) - { - // handle whitespace in string - if constexpr(std::is_same{}) - { - return x; - } - else - { - T result; - std::stringstream ss; - ss.str(x); - ss >> result; - if(ss.fail()) - throw std::runtime_error("Failed to parse '" + x + "' as " + type_name::apply()); - return result; - } - } - - template {} and not is_multi_value{})> - static T apply(const std::string& x) - { - std::ptrdiff_t i; - std::stringstream ss; - ss.str(x); - ss >> i; - if(ss.fail()) - throw std::runtime_error("Failed to parse '" + x + "' as " + type_name::apply()); - return static_cast(i); - } - - template {} and not std::is_enum{})> - static T apply(const std::string& x) - { - T result; - using value_type = typename T::value_type; - result.insert(result.end(), value_parser::apply(x)); - return result; - } -}; - -// version for std::optional object -template -struct value_parser> -{ - static T apply(const std::string& x) { return value_parser::apply(x); } -}; - -struct argument_parser -{ - struct argument - { - using action_function = - std::function&)>; - using validate_function = - std::function&)>; - std::vector flags; - action_function action{}; - std::string type = ""; - std::string help = ""; - std::string metavar = ""; - std::string default_value = ""; - std::string group = ""; - unsigned nargs = 1; - bool required = false; - std::vector validations{}; - - std::string usage(const std::string& flag) const - { - std::stringstream ss; - if(flag.empty()) - { - ss << metavar; - } - else - { - ss << flag; - if(not type.empty()) - ss << " [" << type << "]"; - } - return ss.str(); - } - std::string usage() const - { - if(flags.empty()) - return usage(""); - return usage(flags.front()); - } - }; - - template {})> - std::string as_string_value(const T& x) - { - return to_string_range(x); - } - - template - auto as_string_value(rank<1>, const T& x) -> decltype(to_string(x)) - { - return to_string(x); - } - - template - std::string as_string_value(rank<0>, const T&) - { - throw std::runtime_error("Can't convert to string"); - } - - template {})> - std::string as_string_value(const T& x) - { - return as_string_value(rank<1>{}, x); - } - - template - void operator()(T& x, const std::vector& flags, Fs... fs) - { - arguments.push_back({flags, [&](auto&&, const std::vector& params) { - if(params.empty()) - throw std::runtime_error("Flag with no value."); - if(not is_multi_value{} and params.size() > 1) - throw std::runtime_error("Too many arguments passed."); - x = value_parser::apply(params.back()); - return false; - }}); - - argument& arg = arguments.back(); - arg.type = type_name::apply(); - migraphx::each_args([&](auto f) { f(x, arg); }, fs...); - if(not arg.default_value.empty() and arg.nargs > 0) - arg.default_value = as_string_value(x); - } - - template - void operator()(std::nullptr_t x, std::vector flags, Fs... fs) - { - arguments.push_back({std::move(flags)}); - - argument& arg = arguments.back(); - arg.type = ""; - arg.nargs = 0; - migraphx::each_args([&](auto f) { f(x, arg); }, fs...); - } - - MIGRAPHX_DRIVER_STATIC auto nargs(unsigned n = 1) - { - return [=](auto&&, auto& arg) { arg.nargs = n; }; - } - - MIGRAPHX_DRIVER_STATIC auto required() - { - return [=](auto&&, auto& arg) { arg.required = true; }; - } - - template - MIGRAPHX_DRIVER_STATIC auto write_action(F f) - { - return [=](auto& x, auto& arg) { - arg.action = [&, f](auto& self, const std::vector& params) { - f(self, x, params); - return false; - }; - }; - } - - template - MIGRAPHX_DRIVER_STATIC auto do_action(F f) - { - return [=](auto&, auto& arg) { - arg.nargs = 0; - arg.action = [&, f](auto& self, const std::vector&) { - f(self); - return true; - }; - }; - } - - MIGRAPHX_DRIVER_STATIC auto append() - { - return write_action([](auto&, auto& x, auto& params) { - using type = typename bare::value_type; - std::transform(params.begin(), - params.end(), - std::inserter(x, x.end()), - [](std::string y) { return value_parser::apply(y); }); - }); - } - - template - MIGRAPHX_DRIVER_STATIC auto validate(F f) - { - return [=](const auto& x, auto& arg) { - arg.validations.push_back( - [&, f](auto& self, const std::vector& params) { f(self, x, params); }); - }; - } - - MIGRAPHX_DRIVER_STATIC auto file_exist() - { - return validate([](auto&, auto&, const auto& params) { - if(params.empty()) - throw std::runtime_error("No argument passed."); - if(not fs::exists(params.back())) - throw std::runtime_error("Path does not exist: " + params.back()); - }); - } - - MIGRAPHX_DRIVER_STATIC auto matches(const std::unordered_set& names) - { - return validate([=](auto&, auto&, const auto& params) { - auto invalid_param = std::find_if( - params.begin(), params.end(), [&](const auto& p) { return names.count(p) == 0; }); - if(invalid_param != params.end()) - throw std::runtime_error("Invalid argument: " + *invalid_param + - ". Valid arguments are {" + to_string_range(names) + "}"); - }); - } - - template - argument* find_argument(F f) - { - auto it = std::find_if(arguments.begin(), arguments.end(), f); - if(it == arguments.end()) - return nullptr; - return std::addressof(*it); - } - template - bool has_argument(F f) - { - return find_argument(f) != nullptr; - } - - template - std::vector find_arguments(F f) - { - std::vector result; - for(auto& arg : arguments) - { - if(not f(arg)) - continue; - result.push_back(&arg); - } - return result; - } - - std::vector get_group_arguments(const std::string& group) - { - return find_arguments([&](const auto& arg) { return arg.group == group; }); - } - - std::vector get_required_arguments() - { - return find_arguments([&](const auto& arg) { return arg.required; }); - } - - template - std::vector get_argument_usages(SequenceContainer args) - { - std::vector usage_flags; - std::unordered_set found_groups; - // Remove arguments that belong to a group - auto it = std::remove_if(args.begin(), args.end(), [&](const argument* arg) { - if(arg->group.empty()) - return false; - found_groups.insert(arg->group); - return true; - }); - args.erase(it, args.end()); - transform(found_groups, std::back_inserter(usage_flags), [&](auto&& group) { - std::vector either_flags; - transform(get_group_arguments(group), std::back_inserter(either_flags), [](auto* arg) { - return arg->usage(); - }); - return "(" + join_strings(either_flags, "|") + ")"; - }); - transform(args, std::back_inserter(usage_flags), [&](auto* arg) { return arg->usage(); }); - return usage_flags; - } - - auto show_help(const std::string& msg = "") - { - return do_action([=](auto& self) { - argument* input_argument = - self.find_argument([](const auto& arg) { return arg.flags.empty(); }); - auto required_usages = get_argument_usages(get_required_arguments()); - if(required_usages.empty() and input_argument) - required_usages.push_back(input_argument->metavar); - required_usages.insert(required_usages.begin(), ""); - print_usage(required_usages); - std::cout << std::endl; - if(self.find_argument([](const auto& arg) { return arg.nargs == 0; })) - { - std::cout << color::fg_yellow << "FLAGS:" << color::reset << std::endl; - std::cout << std::endl; - for(auto&& arg : self.arguments) - { - if(arg.nargs != 0) - continue; - const int col_align = 35; - std::string prefix = " "; - int len = 0; - std::cout << color::fg_green; - for(const std::string& a : arg.flags) - { - len += prefix.length() + a.length(); - std::cout << prefix; - std::cout << a; - prefix = ", "; - } - std::cout << color::reset; - int spaces = col_align - len; - if(spaces < 0) - { - std::cout << std::endl; - } - else - { - for(int i = 0; i < spaces; i++) - std::cout << " "; - } - std::cout << arg.help << std::endl; - } - std::cout << std::endl; - } - if(self.find_argument([](const auto& arg) { return arg.nargs != 0; })) - { - std::cout << color::fg_yellow << "OPTIONS:" << color::reset << std::endl; - for(auto&& arg : self.arguments) - { - if(arg.nargs == 0) - continue; - std::cout << std::endl; - std::string prefix = " "; - std::cout << color::fg_green; - if(arg.flags.empty()) - { - std::cout << prefix; - std::cout << arg.metavar; - } - for(const std::string& a : arg.flags) - { - std::cout << prefix; - std::cout << a; - prefix = ", "; - } - std::cout << color::reset; - if(not arg.type.empty()) - { - std::cout << " [" << color::fg_blue << arg.type << color::reset << "]"; - if(not arg.default_value.empty()) - std::cout << " (Default: " << arg.default_value << ")"; - } - std::cout << std::endl; - std::cout << " " << arg.help << std::endl; - } - std::cout << std::endl; - } - if(not msg.empty()) - std::cout << msg << std::endl; - }); - } - - MIGRAPHX_DRIVER_STATIC auto help(const std::string& help) - { - return [=](auto&, auto& arg) { arg.help = help; }; - } - - MIGRAPHX_DRIVER_STATIC auto metavar(const std::string& metavar) - { - return [=](auto&, auto& arg) { arg.metavar = metavar; }; - } - - MIGRAPHX_DRIVER_STATIC auto type(const std::string& type) - { - return [=](auto&, auto& arg) { arg.type = type; }; - } - - MIGRAPHX_DRIVER_STATIC auto group(const std::string& group) - { - return [=](auto&, auto& arg) { arg.group = group; }; - } - - template - MIGRAPHX_DRIVER_STATIC auto set_value(T value) - { - return [=](auto& x, auto& arg) { - arg.nargs = 0; - arg.type = ""; - arg.action = [&, value](auto&, const std::vector&) { - x = value; - return false; - }; - }; - } - - template - void set_exe_name_to(T& x) - { - actions.push_back([&](const auto& self) { x = self.exe_name; }); - } - - void print_try_help() - { - if(has_argument([](const auto& a) { return contains(a.flags, "--help"); })) - { - std::cout << std::endl; - std::cout << "For more information try '" << color::fg_green << "--help" << color::reset - << "'" << std::endl; - } - } - - void print_usage(const std::vector& flags) const - { - std::cout << color::fg_yellow << "USAGE:" << color::reset << std::endl; - std::cout << " " << exe_name << " "; - std::cout << join_strings(flags, " ") << std::endl; - } - - auto spellcheck(const std::vector& inputs) - { - struct result_t - { - const argument* arg = nullptr; - std::string correct = ""; - std::string incorrect = ""; - std::ptrdiff_t distance = std::numeric_limits::max(); - }; - result_t result; - for(const auto& input : inputs) - { - if(input.empty()) - continue; - if(input[0] != '-') - continue; - for(const auto& arg : arguments) - { - for(const auto& flag : arg.flags) - { - if(flag.empty()) - continue; - if(flag[0] != '-') - continue; - std::ptrdiff_t d = levenshtein_distance(flag, input); - if(d < result.distance) - result = result_t{&arg, flag, input, d}; - } - } - } - return result; - } - - bool - run_action(const argument& arg, const std::string& flag, const std::vector& inputs) - { - std::string msg = ""; - try - { - for(const auto& v : arg.validations) - v(*this, inputs); - return arg.action(*this, inputs); - } - catch(const std::exception& e) - { - msg = e.what(); - } - catch(...) - { - msg = "unknown exception"; - } - std::cout << color::fg_red << color::bold << "error: " << color::reset; - auto sc = spellcheck(inputs); - if(sc.distance < 5) - { - std::cout << "Found argument '" << color::fg_yellow << sc.incorrect << color::reset - << "'" - << " which wasn't expected, or isn't valid in this context" << std::endl; - std::cout << " " - << "Did you mean " << color::fg_green << sc.correct << color::reset << "?" - << std::endl; - std::cout << std::endl; - print_usage({sc.arg->usage(sc.correct)}); - } - else - { - const auto& flag_name = flag.empty() ? arg.metavar : flag; - std::cout << "Invalid input to '" << color::fg_yellow; - std::cout << arg.usage(flag_name); - std::cout << color::reset << "'" << std::endl; - std::cout << " " << msg << std::endl; - std::cout << std::endl; - print_usage({arg.usage()}); - } - std::cout << std::endl; - print_try_help(); - return true; - } - - bool parse(std::vector args) - { - std::unordered_map keywords; - for(auto&& arg : arguments) - { - for(auto&& flag : arg.flags) - keywords[flag] = arg.nargs + 1; - } - auto arg_map = - generic_parse(std::move(args), [&](const std::string& x) { return keywords[x]; }); - std::list missing_arguments; - std::unordered_set groups_used; - for(auto&& arg : arguments) - { - bool used = false; - auto flags = arg.flags; - if(flags.empty()) - flags = {""}; - for(auto&& flag : flags) - { - if(arg_map.count(flag) > 0) - { - if(run_action(arg, flag, arg_map[flag])) - return true; - used = true; - } - } - if(used and not arg.group.empty()) - groups_used.insert(arg.group); - if(arg.required and not used) - missing_arguments.push_back(&arg); - } - // Remove arguments from a group that is being used - missing_arguments.remove_if( - [&](const argument* arg) { return groups_used.count(arg->group); }); - if(not missing_arguments.empty()) - { - std::cout << color::fg_red << color::bold << "error: " << color::reset; - std::cout << "The following required arguments were not provided:" << std::endl; - std::cout << " " << color::fg_red - << join_strings(get_argument_usages(std::move(missing_arguments)), " ") - << color::reset << std::endl; - std::cout << std::endl; - auto required_usages = get_argument_usages(get_required_arguments()); - print_usage(required_usages); - print_try_help(); - return true; - } - for(auto&& action : actions) - action(*this); - return false; - } - - void set_exe_name(const std::string& s) { exe_name = s; } - - const std::string& get_exe_name() const { return exe_name; } - - using string_map = std::unordered_map>; - template - static string_map generic_parse(std::vector as, IsKeyword is_keyword) - { - string_map result; - - std::string flag; - bool clear = false; - for(auto&& x : as) - { - auto k = is_keyword(x); - if(k > 0) - { - flag = x; - result[flag]; // Ensure the flag exists - if(k == 1) - flag = ""; - else if(k == 2) - clear = true; - else - clear = false; - } - else - { - result[flag].push_back(x); - if(clear) - flag = ""; - clear = false; - } - } - return result; - } - - private: - std::list arguments; - std::string exe_name = ""; - std::vector> actions; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/command.hpp b/docker/rocm/migraphx/driver/command.hpp deleted file mode 100644 index 7cf825112..000000000 --- a/docker/rocm/migraphx/driver/command.hpp +++ /dev/null @@ -1,116 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_COMMAND_HPP -#define MIGRAPHX_GUARD_RTGLIB_COMMAND_HPP - -#include "argument_parser.hpp" - -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -inline auto& get_commands() -{ - // NOLINTNEXTLINE - static std::unordered_map< - std::string, - std::function args)>> - m; - return m; -} - -template -std::string compute_command_name() -{ - static const std::string& tname = get_type_name(); - auto name = tname.substr(tname.rfind("::") + 2); - if(ends_with(name, "_command")) - name = name.substr(0, name.size() - 8); - if(ends_with(name, "_cmd")) - name = name.substr(0, name.size() - 4); - return name; -} - -template -const std::string& command_name() -{ - static const std::string& name = compute_command_name(); - return name; -} - -template -void run_command(const std::string& exe_name, std::vector args, bool add_help = false) -{ - T x; - argument_parser ap; - ap.set_exe_name(exe_name + " " + command_name()); - if(add_help) - ap(nullptr, {"-h", "--help"}, ap.help("Show help"), ap.show_help()); - x.parse(ap); - if(ap.parse(std::move(args))) - return; - x.run(); -} - -template -int auto_register_command() -{ - auto& m = get_commands(); - m[command_name()] = [](const std::string& exe_name, std::vector args) { - run_command(exe_name, args, true); - }; - return 0; -} - -template -struct command -{ - static const int static_register; - // This typedef ensures that the static member will be instantiated if - // the class itself is instantiated - using static_register_type = - std::integral_constant; -}; - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wglobal-constructors" -#endif - -template -const int command::static_register = auto_register_command(); // NOLINT - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/main.cpp b/docker/rocm/migraphx/driver/main.cpp deleted file mode 100644 index 01d59804e..000000000 --- a/docker/rocm/migraphx/driver/main.cpp +++ /dev/null @@ -1,969 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "verify.hpp" -#include "verify_options.hpp" -#include "argument_parser.hpp" -#include "command.hpp" -#include "mlir.hpp" -#include "precision.hpp" -#include "passes.hpp" -#include "perf.hpp" -#include "models.hpp" -#include "marker_roctx.hpp" - -#include -#include -#ifdef MIGRAPHX_ENABLE_PYTHON -#include -#endif -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -inline std::string get_version() -{ - return "MIGraphX Version: " + std::to_string(MIGRAPHX_VERSION_MAJOR) + "." + - std::to_string(MIGRAPHX_VERSION_MINOR) + "." + std::to_string(MIGRAPHX_VERSION_PATCH) + - "." MIGRAPHX_VERSION_TWEAK; -} - -struct loader -{ - std::string file; - std::string file_type; - unsigned batch = 1; - bool is_nhwc = true; - bool is_test = false; - unsigned trim = 0; - bool optimize = false; - bool mlir = false; - bool skip_unknown_operators = false; - bool brief = false; - std::string output_type; - std::string output; - std::string default_dyn_dim; - std::vector param_dims; - std::vector dim_params; - std::vector dyn_param_dims; - std::vector output_names; - std::vector passes; - - void parse(argument_parser& ap) - { - ap(file, {}, ap.metavar(""), ap.file_exist(), ap.required(), ap.group("input")); - ap(is_test, - {"--test"}, - ap.help("Run a single GEMM to test MIGraphX"), - ap.set_value(true), - ap.group("input")); - ap(file_type, {"--onnx"}, ap.help("Load as onnx"), ap.set_value("onnx")); - ap(file_type, {"--tf"}, ap.help("Load as tensorflow"), ap.set_value("tf")); - ap(file_type, {"--migraphx"}, ap.help("Load as MIGraphX"), ap.set_value("migraphx")); - ap(file_type, {"--migraphx-json"}, ap.help("Load as MIGraphX JSON"), ap.set_value("json")); - ap(batch, - {"--batch"}, - ap.help("For a static model, sets default_dim_value size (commonly batch size). For a " - "dynamic batch model, sets the batch " - "size at runtime.")); - ap(is_nhwc, {"--nhwc"}, ap.help("Treat tensorflow format as nhwc"), ap.set_value(true)); - ap(skip_unknown_operators, - {"--skip-unknown-operators"}, - ap.help("Skip unknown operators when parsing and continue to parse."), - ap.set_value(true)); - ap(is_nhwc, {"--nchw"}, ap.help("Treat tensorflow format as nchw"), ap.set_value(false)); - ap(trim, {"--trim", "-t"}, ap.help("Trim instructions from the end")); - ap(param_dims, - {"--input-dim"}, - ap.help("Dim of a parameter (format: \"@name d1 d2 dn\")"), - ap.append(), - ap.nargs(2)); - ap(dim_params, - {"--dim-param"}, - ap.help("Symbolic parameter dimension name (fixed / dynamic) - " - "(fixed format): \"@dim_param_name\" \"x\" / " - "(dynamic format): \"@dim_param_name\" \"{min:x, max:y, optimals:[o1,o2]}\""), - ap.append(), - ap.nargs(2)); - ap(dyn_param_dims, - {"--dyn-input-dim"}, - ap.help("Dynamic dimensions of a parameter (format: \"@name_1\" \"[{min:x, max:y, " - "optimals:[o1,o2,...]}, dim2,dim3, ...]\", \"@name_2\", ... You can supply a " - "single integer value for a dimension to specify it as fixed."), - ap.append(), - ap.nargs(2)); - ap(default_dyn_dim, - {"--default-dyn-dim"}, - ap.help("Default dynamic dimension (format: \"{min:x, max:y, optimals:[o1,o2]}\").")); - ap(output_names, - {"--output-names"}, - ap.help("Names of node output (format: \"name_1 name_2 name_n\")"), - ap.append(), - ap.nargs(2)); - ap(optimize, {"--optimize", "-O"}, ap.help("Optimize when reading"), ap.set_value(true)); - ap(mlir, {"--mlir"}, ap.help("Offload everything to mlir"), ap.set_value(true)); - ap(passes, {"--apply-pass", "-p"}, ap.help("Passes to apply to model"), ap.append()); - ap(output_type, - {"--graphviz", "-g"}, - ap.help("Print out a graphviz representation."), - ap.set_value("graphviz")); - ap(brief, {"--brief"}, ap.help("Make the output brief."), ap.set_value(true)); - ap(output_type, - {"--cpp"}, - ap.help("Print out the program as C++ program."), - ap.set_value("cpp")); - ap(output_type, - {"--python", "--py"}, - ap.help("Print out the program as python program."), - ap.set_value("py")); - ap(output_type, {"--json"}, ap.help("Print out program as json."), ap.set_value("json")); - ap(output_type, - {"--text"}, - ap.help("Print out program in text format."), - ap.set_value("text")); - ap(output_type, - {"--binary"}, - ap.help("Print out program in binary format."), - ap.set_value("binary")); - ap(output_type, - {"--netron"}, - ap.help("Print out program as Netron readable json."), - ap.set_value("netron")); - ap(output, {"--output", "-o"}, ap.help("Output to file.")); - } - - static auto parse_param_dims(const std::vector& param_dims_info) - { - std::unordered_map> map_input_dims; - std::string name = ""; - for(auto&& x : param_dims_info) - { - if(x[0] == '@') - { - name = x.substr(1); - } - else - { - map_input_dims[name].push_back(value_parser::apply(x)); - } - } - - return map_input_dims; - } - - static auto parse_dyn_dims_json(const std::string& dd_json) - { - // expecting a json string like "[{min:1,max:64,optimals:[1,2,4,8]},3,224,224]" - auto v = from_json_string(convert_to_json(dd_json)); - std::vector dyn_dims; - std::transform(v.begin(), v.end(), std::back_inserter(dyn_dims), [&](auto x) { - if(x.is_object()) - return from_value(x); - auto d = x.template to(); - return migraphx::shape::dynamic_dimension{d, d}; - }); - return dyn_dims; - } - - static auto parse_dyn_dims_map(const std::vector& param_dyn_dims) - { - // expecting vector of strings formatted like - // {"@param_name_0", "dd_json_0", "@param_name_1", "dd_json_1", ...} - std::unordered_map> map_dyn_input_dims; - std::string name = ""; - for(auto&& x : param_dyn_dims) - { - if(x[0] == '@') - { - name = x.substr(1); - } - else - { - map_dyn_input_dims[name] = parse_dyn_dims_json(x); - } - } - return map_dyn_input_dims; - } - - static auto parse_dim_params(const std::vector& dim_params_info) - { - std::unordered_map map_dim_params; - std::string name = ""; - for(auto&& x : dim_params_info) - { - if(x[0] == '@') - { - name = x.substr(1); - } - else - { - if(std::all_of(x.begin(), x.end(), [](char ch) { - return std::isdigit(static_cast(ch)); - })) - map_dim_params[name] = {std::stoul(x), std::stoul(x)}; - else - { - auto dyn_dim = parse_dyn_dims_json(x); - if(dyn_dim.size() != 1) - MIGRAPHX_THROW("dim_param must only specifiy one dimension"); - map_dim_params[name] = dyn_dim.front(); - } - } - } - - return map_dim_params; - } - - static auto parse_output_names(const std::vector& output_names_info) - { - std::vector output_node_names; - std::transform(output_names_info.begin(), - output_names_info.end(), - std::back_inserter(output_node_names), - [&](auto x) { return value_parser::apply(x); }); - - return output_node_names; - } - - tf_options get_tf_options() const - { - auto map_input_dims = parse_param_dims(param_dims); - auto output_node_names = parse_output_names(output_names); - tf_options options; - options.is_nhwc = is_nhwc; - options.batch_size = batch; - options.map_input_dims = map_input_dims; - options.output_node_names = output_node_names; - return options; - } - - onnx_options get_onnx_options() const - { - auto map_input_dims = parse_param_dims(param_dims); - auto map_dyn_input_dims = parse_dyn_dims_map(dyn_param_dims); - auto map_dim_params = parse_dim_params(dim_params); - onnx_options options; - if(default_dyn_dim.empty()) - { - options.default_dim_value = batch; - } - else - { - auto v = from_json_string(convert_to_json(default_dyn_dim)); - options.default_dyn_dim_value = from_value(v); - } - options.skip_unknown_operators = skip_unknown_operators; - options.print_program_on_error = true; - options.map_input_dims = map_input_dims; - options.map_dyn_input_dims = map_dyn_input_dims; - options.dim_params = map_dim_params; - return options; - } - - static std::string get_file_type(const std::string& file) - { - if(ends_with(file, ".onnx")) - return "onnx"; - else if(ends_with(file, ".pb")) - return "tf"; - else if(ends_with(file, ".json")) - return "json"; - else if(ends_with(file, ".py")) - return "py"; - else - return "migraphx"; - } - - program load() - { - program p; - if(is_test) - { - p = test_gemm(); - } - else - { - if(file_type.empty()) - { - file_type = get_file_type(file); - } - std::cout << "Reading: " << file << std::endl; - if(file_type == "onnx") - { - p = parse_onnx(file, get_onnx_options()); - } - else if(file_type == "tf") - { - p = parse_tf(file, get_tf_options()); - } - else if(file_type == "json") - { - file_options options; - options.format = "json"; - p = migraphx::load(file, options); - } -#ifdef MIGRAPHX_ENABLE_PYTHON - else if(file_type == "py") - { - p = migraphx::load_py(file); - } -#endif - else if(file_type == "migraphx") - { - p = migraphx::load(file); - } - } - if(trim > 0) - { - auto* mm = p.get_main_module(); - auto last = std::prev(mm->end(), trim); - mm->remove_instructions(last, mm->end()); - } - // Remove unused variable when exporting to cpp - if(output_type == "cpp") - migraphx::run_passes(*p.get_main_module(), {migraphx::dead_code_elimination{}}); - if(optimize) - { - migraphx::run_passes(*p.get_main_module(), - { - migraphx::eliminate_identity{}, - migraphx::dead_code_elimination{}, - migraphx::simplify_algebra{}, - migraphx::dead_code_elimination{}, - migraphx::simplify_reshapes{}, - migraphx::dead_code_elimination{}, - migraphx::propagate_constant{}, - migraphx::dead_code_elimination{}, - migraphx::eliminate_pad{}, - migraphx::dead_code_elimination{}, - }); - } - if(not passes.empty()) - migraphx::run_passes(p, get_passes(passes)); - if(mlir) - offload_to_mlir(p); - return p; - } - - static void write(std::ostream& os, const std::vector& buffer) - { - os.write(buffer.data(), buffer.size()); - } - - void save(const program& p) const - { - auto* os = &std::cout; - std::ofstream fs; - if(not output.empty()) - { - fs.open(output, std::ios::binary); - os = &fs; - } - - std::string type = output_type; - if(type.empty()) - { - if(output.empty()) - type = "text"; - else - type = "binary"; - } - - if(type == "py") - p.print_py(*os); - else if(type == "cpp") - p.print_cpp(*os); - else if(type == "graphviz") - p.print_graph(*os, brief); - else if(type == "text") - *os << p << std::endl; - else if(type == "json") - *os << to_json_string(p.to_value()) << std::endl; - else if(type == "binary") - write(*os, save_buffer(p)); - else if(type == "netron") - *os << make_netron_output(p) << std::endl; - } -}; - -struct program_params -{ - std::vector fill0{}; - std::vector fill1{}; - void parse(argument_parser& ap) - { - ap(fill0, {"--fill0"}, ap.help("Fill parameter with 0s"), ap.append(), ap.nargs(2)); - ap(fill1, {"--fill1"}, ap.help("Fill parameter with 1s"), ap.append(), ap.nargs(2)); - } - - auto generate(const program& p, const target& t, bool offload, unsigned batch) - { - parameter_map m; - auto param_shapes = p.get_parameter_shapes(); - std::unordered_map static_param_shapes; - std::transform( - param_shapes.cbegin(), - param_shapes.cend(), - std::inserter(static_param_shapes, static_param_shapes.end()), - [&](const auto& x) { return std::make_pair(x.first, x.second.to_static(batch)); }); - for(auto&& s : fill0) - m[s] = fill_argument(static_param_shapes.at(s), 0); - for(auto&& s : fill1) - m[s] = fill_argument(static_param_shapes.at(s), 1); - fill_param_map(m, static_param_shapes, t, offload); - return m; - } -}; - -struct compiler_target -{ -#ifdef HAVE_GPU - std::string target_name = "gpu"; -#elif defined(HAVE_CPU) - std::string target_name = "cpu"; -#elif defined(HAVE_FPGA) - std::string target_name = "fpga"; -#else - std::string target_name = "ref"; -#endif - - void parse(argument_parser& ap) - { - ap(target_name, {"--gpu"}, ap.help("Compile on the gpu"), ap.set_value("gpu")); - ap(target_name, {"--cpu"}, ap.help("Compile on the cpu"), ap.set_value("cpu")); - ap(target_name, - {"--ref"}, - ap.help("Compile on the reference implementation"), - ap.set_value("ref")); - } - - target get_target() const { return make_target(target_name); } -}; - -struct compiler -{ - loader l; - program_params parameters; - compiler_target ct; - compile_options co; - bool to_fp16 = false; - bool to_bf16 = false; - bool to_fp8 = false; - bool to_int8 = false; - bool to_int4 = false; - - std::vector fill0; - std::vector fill1; - void parse(argument_parser& ap) - { - l.parse(ap); - parameters.parse(ap); - ct.parse(ap); - ap(co.offload_copy, - {"--enable-offload-copy"}, - ap.help("Enable implicit offload copying"), - ap.set_value(true)); - ap(co.fast_math, - {"--disable-fast-math"}, - ap.help("Disable fast math optimization"), - ap.set_value(false)); - ap(co.exhaustive_tune, - {"--exhaustive-tune"}, - ap.help("Exhastively search for best tuning parameters for kernels"), - ap.set_value(true)); - ap(to_fp16, {"--fp16"}, ap.help("Quantize for fp16"), ap.set_value(true)); - ap(to_bf16, {"--bf16"}, ap.help("Quantize for bf16"), ap.set_value(true)); - ap(to_int8, {"--int8"}, ap.help("Quantize for int8"), ap.set_value(true)); - ap(to_fp8, {"--fp8"}, ap.help("Quantize for fp8"), ap.set_value(true)); - ap(to_int4, {"--int4-weights"}, ap.help("Quantize weights for int4"), ap.set_value(true)); - } - - auto params(const program& p) - { - return parameters.generate(p, ct.get_target(), co.offload_copy, l.batch); - } - - auto host_params(const program& p) - { - return parameters.generate(p, ct.get_target(), true, l.batch); - } - - program compile() - { - auto p = l.load(); - // Dont compile if its already been compiled - - if(p.is_compiled()) - { - if(ct.target_name == "gpu") - { - if(is_offload_copy_set(p) and not co.offload_copy) - { - std::cout - << "[WARNING]: MIGraphX program was likely compiled with offload_copy " - "set, Try " - "passing " - "`--enable-offload-copy` if program run fails.\n"; - } - else if(not is_offload_copy_set(p) and co.offload_copy) - { - std::cout << "[WARNING]: MIGraphX program was likely compiled without " - "offload_copy set, Try " - "removing " - "`--enable-offload-copy` if program run " - "fails.\n"; - } - } - - return p; - } - auto t = ct.get_target(); - if(to_fp16) - { - quantize_fp16(p); - } - if(to_bf16) - { - quantize_bf16(p); - } - if(to_int8) - { - quantize_int8(p, t, {host_params(p)}); - } - if(to_fp8) - { - quantize_fp8(p, t, {host_params(p)}); - } - if(to_int4) - { - quantize_int4_weights(p); - } - p.compile(t, co); - l.save(p); - return p; - } -}; - -struct read : command -{ - loader l; - void parse(argument_parser& ap) { l.parse(ap); } - - void run() - { - auto p = l.load(); - l.save(p); - } -}; - -struct params : command -{ - loader l; - void parse(argument_parser& ap) { l.parse(ap); } - - void run() - { - auto p = l.load(); - for(auto&& param : p.get_parameter_shapes()) - std::cout << param.first << ": " << param.second << std::endl; - } -}; - -struct verify : command -{ - compiler c; - std::optional rms_tol; - std::optional atol; - std::optional rtol; - bool per_instruction = false; - bool reduce = false; - bool bisect = false; - verify_options vo; - void parse(argument_parser& ap) - { - c.parse(ap); - ap(rms_tol, {"--rms-tol"}, ap.help("Tolerance for the RMS error")); - ap(atol, {"--atol"}, ap.help("Tolerance for the elementwise absolute difference")); - ap(rtol, {"--rtol"}, ap.help("Tolerance for the elementwise relative difference")); - ap(per_instruction, - {"-i", "--per-instruction"}, - ap.help("Verify each instruction"), - ap.set_value(true)); - ap(reduce, {"-r", "--reduce"}, ap.help("Reduce program and verify"), ap.set_value(true)); - ap(bisect, {"-b", "--bisect"}, ap.help("Bisect program and verify"), ap.set_value(true)); - ap(vo.ref_use_double, - {"--ref-use-double"}, - ap.help("Convert floating point values to double on ref"), - ap.set_value(true)); - } - - void run() - { - auto p = c.l.load(); - c.l.save(p); - std::cout << p << std::endl; - - auto t = c.ct.get_target(); - auto m = c.parameters.generate(p, t, true, c.l.batch); - - if(c.to_fp16) - { - vo.quantize = precision::fp16; - } - if(c.to_bf16) - { - vo.quantize = precision::bf16; - } - if(c.to_int8) - { - vo.quantize = precision::int8; - } - - auto tols = get_tolerances(p, vo, rms_tol, atol, rtol); - std::cout << "rms_tol: " << tols.rms_tol << std::endl; - std::cout << "atol: " << tols.atol << std::endl; - std::cout << "rtol: " << tols.rtol << std::endl; - - if(per_instruction) - { - verify_instructions(p, t, c.co, vo, tols); - } - else if(reduce) - { - verify_reduced_program(p, t, c.co, vo, m, tols); - } - else if(bisect) - { - verify_bisected_program(p, t, c.co, vo, m, tols); - } - else - { - verify_program(c.l.file, p, t, c.co, vo, m, tols); - } - } -}; - -struct compile : command -{ - compiler c; - void parse(argument_parser& ap) { c.parse(ap); } - - void run() - { - std::cout << "Compiling ... " << std::endl; - c.compile(); - } -}; - -struct run_cmd : command -{ - compiler c; - void parse(argument_parser& ap) { c.parse(ap); } - - void run() - { - std::cout << "Compiling ... " << std::endl; - auto p = c.compile(); - std::cout << "Allocating params ... " << std::endl; - auto m = c.params(p); - p.eval(m); - std::cout << p << std::endl; - } -}; - -struct time_cmd : command -{ - compiler c; - unsigned n = 100; - void parse(argument_parser& ap) - { - ap(n, {"--iterations", "-n"}, ap.help("Number of iterations to run.")); - c.parse(ap); - } - - void run() - { - std::cout << "Compiling ... " << std::endl; - auto p = c.compile(); - std::cout << "Allocating params ... " << std::endl; - auto m = c.params(p); - std::cout << "Running ... " << std::endl; - double t = time_run(p, m, n); - std::cout << "Total time: " << t << "ms" << std::endl; - } -}; - -struct perf : command -{ - compiler c; - unsigned n = 100; - bool detailed = false; - void parse(argument_parser& ap) - { - c.parse(ap); - ap(n, {"--iterations", "-n"}, ap.help("Number of iterations to run for perf report")); - ap(detailed, - {"--detailed", "-d"}, - ap.help("Show a more detailed summary report"), - ap.set_value(true)); - } - - void run() - { - std::cout << "Compiling ... " << std::endl; - auto p = c.compile(); - std::cout << "Allocating params ... " << std::endl; - auto m = c.params(p); - std::cout << "Running performance report ... " << std::endl; - p.perf_report(std::cout, n, m, c.l.batch, detailed); - } -}; - -struct roctx : command -{ - compiler c; - void parse(argument_parser& ap) { c.parse(ap); } - - void run() - { - std::cout << "Compiling ... " << std::endl; - auto p = c.compile(); - std::cout << "Allocating params ... " << std::endl; - auto m = c.params(p); - std::cout << "rocTX:\tLoading rocTX library..." << std::endl; - auto rtx = create_marker_roctx(); - p.mark(m, std::move(rtx)); - } -}; - -struct op : command -{ - bool show_ops = false; - std::string op_name{}; - void parse(argument_parser& ap) - { - ap(op_name, {}, ap.metavar("")); - ap(show_ops, - {"--list", "-l"}, - ap.help("List all the operators of MIGraphX"), - ap.set_value(true)); - } - void run() const - { - if(show_ops) - { - for(const auto& name : get_operators()) - std::cout << name << std::endl; - } - else - { - auto op = load_op(op_name); - std::cout << op_name << ": " << std::endl; - std::cout << to_pretty_json_string(op.to_value()) << std::endl; - } - } -}; - -struct onnx : command -{ - bool show_ops = false; - void parse(argument_parser& ap) - { - ap(show_ops, - {"--list", "-l"}, - ap.help("List all onnx operators supported by MIGraphX"), - ap.set_value(true)); - } - void run() const - { - if(show_ops) - { - for(const auto& name : get_onnx_operators()) - std::cout << name << std::endl; - } - } -}; - -struct tf : command -{ - bool show_ops = false; - void parse(argument_parser& ap) - { - ap(show_ops, - {"--list", "-l"}, - ap.help("List all tf operators supported by MIGraphX"), - ap.set_value(true)); - } - void run() const - { - if(show_ops) - { - for(const auto& name : get_tf_operators()) - std::cout << name << std::endl; - } - } -}; - -struct main_command -{ - static std::string get_command_help(const std::string& title = colorize(color::fg_yellow, - "COMMANDS:")) - { - std::string result = title + "\n"; - std::vector commands(get_commands().size()); - std::transform(get_commands().begin(), - get_commands().end(), - commands.begin(), - [](const auto& p) { return colorize(color::fg_green, p.first); }); - std::sort(commands.begin(), commands.end()); - return std::accumulate(commands.begin(), commands.end(), result, [](auto r, auto&& s) { - return r + " " + s + "\n"; - }); - } - void parse(argument_parser& ap) - { - std::string version_str = get_version(); - ap(wrong_commands, {}, ap.metavar(""), ap.append()); - ap(nullptr, {"-h", "--help"}, ap.help("Show help"), ap.show_help(get_command_help())); - ap(nullptr, - {"-v", "--version"}, - ap.help("Show MIGraphX version"), - ap.show_help(version_str)); - ap(nullptr, {"--ort-sha"}, ap.help("Show MIGraphX onnx runtime SHA")); - - // Trim command off of exe name - ap.set_exe_name(ap.get_exe_name().substr(0, ap.get_exe_name().size() - 5)); - ap.set_exe_name_to(exe_name); - } - - std::vector wrong_commands{}; - std::string exe_name = ""; - - void run() - { - std::cout << color::fg_red << color::bold << "error: " << color::reset; - auto it = std::find_if(wrong_commands.begin(), wrong_commands.end(), [](const auto& c) { - return get_commands().count(c) > 0; - }); - if(it == wrong_commands.end()) - { - std::cout << "'" << color::fg_yellow << wrong_commands.front() << color::reset - << "' is not a valid command." << std::endl; - std::cout << get_command_help("Available commands:"); - } - else - { - std::cout << "command '" << color::fg_yellow << *it << color::reset - << "' must be first argument" << std::endl; - std::cout << std::endl; - - std::cout << color::fg_yellow << "USAGE:" << color::reset << std::endl; - std::cout << " " << exe_name << " " << *it << " " << std::endl; - } - std::cout << std::endl; - } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -using namespace migraphx::driver; // NOLINT -int main(int argc, const char* argv[], const char* envp[]) -{ - std::vector args(argv + 1, argv + argc); - // no argument, print the help infomration by default - if(args.empty()) - { - args.push_back("-h"); - } - - auto&& m = get_commands(); - auto cmd = args.front(); - - if(cmd == "--ort-sha") - { - std::cout << MIGRAPHX_ORT_SHA1 << std::endl; - return 0; - } - if(cmd == "-v" or cmd == "--version") - { - std::cout << get_version() << std::endl; - return 0; - } - - if(m.count(cmd) > 0) - { - std::string driver_invocation = - std::string(argv[0]) + " " + migraphx::to_string_range(args, " "); - std::cout << "Running [ " << get_version() << " ]: " << driver_invocation << std::endl; - - for(const char** env = envp; *env != nullptr; ++env) - { - std::string env_var(*env); - size_t pos = env_var.find('='); - if(pos != std::string::npos) - { - std::string key = env_var.substr(0, pos); - if(key.find("MIGRAPHX") != std::string::npos) - { - std::cout << env_var << std::endl; - } - } - } - - m.at(cmd)(argv[0], - {args.begin() + 1, args.end()}); // run driver command found in commands map - - std::cout << "[ " << get_version() << " ] Complete: " << driver_invocation << std::endl; - } - else - { - run_command(argv[0], args); - } - - return 0; -} diff --git a/docker/rocm/migraphx/driver/marker_roctx.cpp b/docker/rocm/migraphx/driver/marker_roctx.cpp deleted file mode 100644 index 1011af4bf..000000000 --- a/docker/rocm/migraphx/driver/marker_roctx.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "marker_roctx.hpp" - -#include -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -class marker_roctx -{ - std::function sym_roctx_mark; - std::function sym_roctx_range_start; - std::function sym_roctx_range_stop; - - std::function sym_roctx_range_push; - std::function sym_roctx_range_pop; - - uint64_t range_id = 0; - - public: - marker_roctx() - { - dynamic_loader lib = migraphx::dynamic_loader{"libroctx64.so"}; - sym_roctx_mark = lib.get_function("roctxMarkA"); - sym_roctx_range_start = lib.get_function("roctxRangeStartA"); - sym_roctx_range_stop = lib.get_function("roctxRangeStop"); - - sym_roctx_range_push = lib.get_function("roctxRangePushA"); - sym_roctx_range_pop = lib.get_function("roctxRangePop"); - - sym_roctx_mark("rocTX marker created."); - } - - void mark_start(instruction_ref ins_ref) - { - std::string text = "Marker start: " + ins_ref->name(); - sym_roctx_range_push(text.c_str()); - } - void mark_stop(instruction_ref) { sym_roctx_range_pop(); } - void mark_start(const program&) { range_id = sym_roctx_range_start("0"); } - void mark_stop(const program&) { sym_roctx_range_stop(range_id); } -}; - -marker create_marker_roctx() { return marker_roctx(); } -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/marker_roctx.hpp b/docker/rocm/migraphx/driver/marker_roctx.hpp deleted file mode 100644 index 1a25fe1a9..000000000 --- a/docker/rocm/migraphx/driver/marker_roctx.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MARKER_ROCTX_HPP -#define MIGRAPHX_GUARD_RTGLIB_MARKER_ROCTX_HPP - -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -marker create_marker_roctx(); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/mlir.cpp b/docker/rocm/migraphx/driver/mlir.cpp deleted file mode 100644 index 1a90f0491..000000000 --- a/docker/rocm/migraphx/driver/mlir.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include "mlir.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -void offload_to_mlir(program& p) -{ - auto* mm = p.get_main_module(); - auto* mlirm = p.create_module("mlir"); - mlirm->set_bypass(); - std::vector inputs; - copy_if(iterator_for(*mm), std::back_inserter(inputs), [&](instruction_ref ins) { - if(ins->name() == "@param") - return true; - if(ins->name() == "@literal") - return ins->get_shape().elements() != 1; - return false; - }); - - std::unordered_map map_ins; - std::size_t n = 0; - for(auto ins : inputs) - { - map_ins[ins] = mlirm->add_parameter(param_name(n++), ins->get_shape().as_standard()); - } - - auto mlir_last = mlirm->add_instructions(mm, &map_ins); - mlirm->add_return(mlir_last); - - auto last = std::prev(mm->end()); - auto mlir_op = mm->insert_instruction(last, make_op("gpu::mlir_op"), inputs, {mlirm}); - if(mlir_last.size() > 1) - { - std::vector outputs; - transform(range(mlir_last.size()), std::back_inserter(outputs), [&](auto i) { - return mm->insert_instruction(last, make_op("get_tuple_elem", {{"index", i}}), mlir_op); - }); - mm->replace_return(outputs); - } - else - { - mm->replace_return({mlir_op}); - } - run_passes(*mm, {dead_code_elimination{}}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/mlir.hpp b/docker/rocm/migraphx/driver/mlir.hpp deleted file mode 100644 index e9f874d0b..000000000 --- a/docker/rocm/migraphx/driver/mlir.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef MIGRAPHX_GUARD_DRIVER_MLIR_HPP -#define MIGRAPHX_GUARD_DRIVER_MLIR_HPP - -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -void offload_to_mlir(program& p); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx -#endif // MIGRAPHX_GUARD_DRIVER_MLIR_HPP diff --git a/docker/rocm/migraphx/driver/models.cpp b/docker/rocm/migraphx/driver/models.cpp deleted file mode 100644 index a2b002123..000000000 --- a/docker/rocm/migraphx/driver/models.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "models.hpp" -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -migraphx::program test_gemm() -{ - migraphx::program p; - auto* mm = p.get_main_module(); - auto a = mm->add_parameter("a", migraphx::shape{migraphx::shape::float_type, {4, 5}}); - auto b = mm->add_parameter("b", migraphx::shape{migraphx::shape::float_type, {5, 3}}); - mm->add_instruction(migraphx::make_op("dot"), a, b); - return p; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/models.hpp b/docker/rocm/migraphx/driver/models.hpp deleted file mode 100644 index 8d4209285..000000000 --- a/docker/rocm/migraphx/driver/models.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -migraphx::program test_gemm(); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/passes.cpp b/docker/rocm/migraphx/driver/passes.cpp deleted file mode 100644 index cd21edaee..000000000 --- a/docker/rocm/migraphx/driver/passes.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "passes.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -std::unordered_map create_passes_lookup() -{ - std::unordered_map result; - // clang-format off - std::initializer_list passes = { - auto_contiguous{}, - dead_code_elimination{}, - eliminate_allocation{}, - eliminate_common_subexpression{}, - eliminate_concat{}, - eliminate_contiguous{}, - eliminate_data_type{}, - eliminate_identity{}, - eliminate_pad{}, - fuse_pointwise{}, - fuse_reduce{}, - inline_module{}, - insert_pad{}, - normalize_ops{}, - optimize_module{}, - promote_literals{}, - propagate_constant{}, - rewrite_gelu{}, - rewrite_pooling{}, - rewrite_quantization{}, - rewrite_rnn{}, - simplify_algebra{}, - simplify_dyn_ops{}, - simplify_qdq{}, - simplify_reshapes{}, - }; - // clang-format on - for(const auto& pass : passes) - result[pass.name()] = pass; - result["eliminate_dead_code"] = dead_code_elimination{}; - return result; -} - -std::vector get_passes(const std::vector& names) -{ - std::vector result; - static const std::unordered_map lookup = create_passes_lookup(); - std::transform( - names.begin(), names.end(), std::back_inserter(result), [](const std::string& name) { - if(not contains(lookup, name)) - MIGRAPHX_THROW("Unknown pass: " + name); - return lookup.at(name); - }); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/passes.hpp b/docker/rocm/migraphx/driver/passes.hpp deleted file mode 100644 index 41dc1c7e3..000000000 --- a/docker/rocm/migraphx/driver/passes.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_DRIVER_PASSES_HPP -#define MIGRAPHX_GUARD_DRIVER_PASSES_HPP - -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -std::vector get_passes(const std::vector& names); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/perf.cpp b/docker/rocm/migraphx/driver/perf.cpp deleted file mode 100644 index 47dbe0fc4..000000000 --- a/docker/rocm/migraphx/driver/perf.cpp +++ /dev/null @@ -1,157 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "perf.hpp" - -#include -#include -#include -#include -#include -#include -#ifdef HAVE_GPU -#include -#endif - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -using milliseconds = std::chrono::duration; - -template -auto get_hash(const T& x) -{ - return std::hash{}(x); -} - -parameter_map fill_param_map(parameter_map& m, - const std::unordered_map& param_shapes, - const target& t, - bool offload) -{ - for(auto&& x : param_shapes) - { - argument& arg = m[x.first]; - if(arg.empty()) - { - assert(not x.second.dynamic()); - arg = generate_argument(x.second, get_hash(x.first), random_mode::random); - } - if(not offload) - arg = t.copy_to(arg); - } - return m; -} - -parameter_map create_param_map(const program& p, const target& t, bool offload) -{ - parameter_map m; - for(auto&& x : p.get_parameter_shapes()) - { - auto arg = generate_argument(x.second, get_hash(x.first), random_mode::random); - if(offload) - m[x.first] = arg; - else - m[x.first] = t.copy_to(arg); - } - return m; -} - -parameter_map create_param_map(const program& p, bool gpu) -{ - parameter_map m; - for(auto&& x : p.get_parameter_shapes()) - { -#ifdef HAVE_GPU - if(gpu) - m[x.first] = - gpu::to_gpu(generate_argument(x.second, get_hash(x.first), random_mode::random)); - else -#else - (void)gpu; -#endif - m[x.first] = generate_argument(x.second, get_hash(x.first), random_mode::random); - } - return m; -} - -target get_target(bool gpu) -{ - if(gpu) - return make_target("gpu"); - else - return make_target("cpu"); -} - -bool is_offload_copy_set(const program& p) -{ - assert(p.is_compiled()); - const module* mm = p.get_main_module(); - std::vector param_names = mm->get_parameter_names(); - std::unordered_set param_ins; - std::transform(param_names.begin(), - param_names.end(), - std::inserter(param_ins, param_ins.begin()), - [&](const auto& i) { return mm->get_parameter(i); }); - for(const auto& i : *mm) - { - if(i.name() == "hip::copy_to_gpu") - { - auto copy_arg = instruction::get_output_alias(i.inputs().front(), true); - param_ins.erase(copy_arg); - } - else if(i.name() == "@return") - { - auto return_args = i.inputs(); - for(const auto& j : return_args) - { - auto alias_ins = instruction::get_output_alias(j, true); - if((alias_ins->name() == "@param" and param_ins.erase(alias_ins) == 0) or - (alias_ins->name() != "hip::copy_from_gpu")) - return false; - } - } - } - return param_ins.empty(); -} - -double time_run(const program& p, const parameter_map& m, int n) -{ - // Run once without timing - p.eval(m); - p.finish(); - double total = time([&] { - for(auto i : range(n)) - { - (void)i; - p.eval(m); - } - p.finish(); - }); - return total / n; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/perf.hpp b/docker/rocm/migraphx/driver/perf.hpp deleted file mode 100644 index 4602be381..000000000 --- a/docker/rocm/migraphx/driver/perf.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PERF_HPP -#define MIGRAPHX_GUARD_RTGLIB_PERF_HPP - -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -parameter_map fill_param_map(parameter_map& m, - const std::unordered_map& param_shapes, - const target& t, - bool offload = false); -parameter_map create_param_map(const program& p, const target& t, bool offload = false); - -parameter_map fill_param_map(parameter_map& m, const program& p, bool gpu); -parameter_map create_param_map(const program& p, bool gpu = true); -target get_target(bool gpu); -/** - * @brief Checks if MIGraphX program compiled for "GPU" has offload_copy set of not. This is - intended to print a HINT for the users and would not always correctly classify compiled program as - with or without offload_copy in all cases. - - * @param p Compiled MIGraphX program for GPU backend - * @return true if program is classified as compiled with "offload_copy" set - */ -bool is_offload_copy_set(const program& p); - -double time_run(const program& p, const parameter_map& m, int n = 100); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/precision.hpp b/docker/rocm/migraphx/driver/precision.hpp deleted file mode 100644 index 9ed1f402f..000000000 --- a/docker/rocm/migraphx/driver/precision.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PRECISION_HPP -#define MIGRAPHX_GUARD_RTGLIB_PRECISION_HPP - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -enum class precision -{ - fp32, - fp16, - bf16, - int8 -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/verify.cpp b/docker/rocm/migraphx/driver/verify.cpp deleted file mode 100644 index 14f9e71f7..000000000 --- a/docker/rocm/migraphx/driver/verify.cpp +++ /dev/null @@ -1,340 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "verify.hpp" -#include "perf.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Gives tolerances based on user input (`rms_tol`, `atol`, `rtol` parameters) and defaults. - * Sets to fp16 tolerances if `quantize` input is fp16 or any fp16 instruction in found in the - * model. - */ -verify::tolerance get_tolerances(const program& p, - verify_options vo, - std::optional rms_tol, - std::optional atol, - std::optional rtol) -{ - bool has_16bit = any_of(p.get_modules(), [](auto&& m) { - return any_of(*m, [](auto&& ins) { - return (ins.get_shape().type() == shape::half_type or - ins.get_shape().type() == shape::bf16_type); - }); - }); - migraphx::verify::tolerance result{}; - if(has_16bit or vo.quantize == precision::fp16 or vo.quantize == precision::bf16) - { - result.rms_tol = 8e-2; - result.atol = 4e-2; - result.rtol = 4e-2; - } - if(rms_tol) - { - result.rms_tol = *rms_tol; - } - if(atol) - { - result.atol = *atol; - } - if(rtol) - { - result.rtol = *rtol; - } - return result; -} - -std::vector run_ref(program p, - const compile_options& options, - const verify_options& vo, - const parameter_map& inputs) -{ - if(vo.ref_use_double) - { - run_passes(p, {fp_to_double{}}); - } - p.compile(migraphx::make_target("ref"), options); - auto out = p.eval(inputs); - std::cout << p << std::endl; - return out; -} - -std::vector run_target(program p, - const target& t, - const compile_options& options, - const verify_options& vo, - const parameter_map& inputs) -{ - if(vo.quantize == precision::fp16) - { - quantize_fp16(p); - } - if(vo.quantize == precision::bf16) - { - quantize_bf16(p); - } - p.compile(t, options); - - parameter_map m; - for(auto&& x : p.get_parameter_shapes()) - { - auto arg = inputs.count(x.first) == 0 ? generate_argument(x.second) : inputs.at(x.first); - m[x.first] = options.offload_copy ? arg : t.copy_to(arg); - } - auto gpu_out = p.eval(m); - std::vector output(gpu_out.size()); - std::cout << p << std::endl; - std::transform(gpu_out.begin(), gpu_out.end(), output.begin(), [&](auto& argu) { - return options.offload_copy ? argu : t.copy_from(argu); - }); - return output; -} - -bool verify_program(const std::string& name, - const program& p, - const target& t, - compile_options options, - verify_options vo, - const parameter_map& inputs, - verify::tolerance tols) -{ - auto ref_outs = run_ref(p, options, vo, inputs); - auto target_outs = run_target(p, t, options, vo, inputs); - - std::size_t output_num = ref_outs.size(); - bool passed = true; - for(std::size_t i = 0; i < output_num; ++i) - { - if(ref_outs[i].get_shape().type() != target_outs[i].get_shape().type() or - ref_outs[i].get_shape().lens() != target_outs[i].get_shape().lens()) - { - std::cout << "FAILED: " << name << std::endl; - std::cout << "Shape mismatch {" << ref_outs[i].get_shape() << "} != {" - << target_outs[i].get_shape() << "}" << std::endl; - } - else - { - passed &= verify_args(name, target_outs[i], verify::expected{ref_outs[i]}, tols); - } - } - if(passed) - std::cout << "MIGraphX verification passed successfully." << std::endl; - return passed; -} - -void verify_instructions(const program& prog, - const target& t, - compile_options options, - verify_options vo, - verify::tolerance tols) -{ - const auto* mm_prog = prog.get_main_module(); - for(auto&& ins : (*mm_prog)) - { - if(ins.name().front() == '@') - continue; - if(ins.name() == "broadcast") - continue; - if(ins.name() == "transpose") - continue; - if(ins.name() == "reshape") - continue; - if(ins.name() == "undefined") - continue; - program p; - auto* mm_p = p.get_main_module(); - std::vector inputs; - for(auto&& arg : ins.inputs()) - { - if(arg->name() == "@literal") - inputs.push_back(mm_p->add_literal(arg->get_literal())); - else - inputs.push_back( - mm_p->add_parameter(std::to_string(inputs.size()), arg->get_shape())); - } - mm_p->add_instruction(ins.get_operator(), inputs); - try - { - std::cout << "Verify: " << ins.name() << std::endl; - std::cout << p << std::endl; - verify_program(ins.name(), p, t, options, vo, create_param_map(p, false), tols); - } - catch(...) - { - std::cout << "Instruction " << ins.name() << " threw an exception." << std::endl; - throw; - } - } -} - -bool verify_reduced(program p, - int n, - const target& t, - compile_options options, - verify_options vo, - const parameter_map& inputs, - verify::tolerance tols) -{ - auto* mm = p.get_main_module(); - auto last = std::prev(mm->end(), n); - mm->remove_instructions(last, mm->end()); - std::cout << "Verify: " << n << std::endl; - std::cout << p << std::endl; - try - { - return verify_program(std::to_string(n), p, t, options, vo, inputs, tols); - } - catch(const std::exception& e) - { - std::cout << "FAILED: " << n << std::endl; - std::cout << "Exception: " << e.what() << std::endl; - return false; - } -} - -void verify_reduced_program(const program& p, - const target& t, - compile_options options, - verify_options vo, - const parameter_map& inputs, - verify::tolerance tols) -{ - const auto* mm = p.get_main_module(); - auto n = std::distance(mm->begin(), mm->end()); - std::cout << "Verify steps: " << n << std::endl; - for(std::size_t i = 1; i < n; i++) - { - auto last = std::prev(mm->end(), i + 1); - if(contains({"@literal", "@param"}, last->name())) - { - std::cout << "Skip: " << i << std::endl; - continue; - } - verify_reduced(p, i, t, options, vo, inputs, tols); - } -} - -static std::unordered_map accumulate_weights(instruction_ref last) -{ - std::unordered_map weights; - fix([&](auto self, auto ins) -> std::size_t { - if(not contains(weights, ins)) - { - if(ins->can_eval()) - return 0; - std::size_t weight = 1; - weights[ins] = std::accumulate( - ins->inputs().begin(), - ins->inputs().end(), - weight, - [&](std::size_t w, instruction_ref i) -> std::size_t { return w + self(i); }); - } - return weights[ins]; - })(last); - return weights; -} - -static optional -get_parent(const std::unordered_map& weights, instruction_ref ins) -{ - if(ins->inputs().empty()) - return nullopt; - auto next = std::max_element(ins->inputs().begin(), - ins->inputs().end(), - by(std::less<>{}, [&](instruction_ref input) -> std::size_t { - if(not contains(weights, input)) - return 0; - return weights.at(input); - })); - return *next; -} - -static std::vector find_trim_instructions(const module& m) -{ - std::vector result; - auto last = std::prev(m.end()); - auto weights = accumulate_weights(last); - auto next = get_parent(weights, last); - std::size_t i = 0; - while(auto parent = get_parent(weights, *next)) - { - i += std::distance(*parent, *next); - result.push_back(i + 1); - next = parent; - } - return result; -} - -void verify_bisected_program(const program& p, - const target& t, - compile_options options, - verify_options vo, - const parameter_map& inputs, - verify::tolerance tols) -{ - const auto* mm = p.get_main_module(); - - std::vector trims = find_trim_instructions(*mm); - std::int64_t right = trims.size(); - std::int64_t left = 0; - std::int64_t failed = -1; - - while(left <= right) - { - std::int64_t mid = left + (right - left) / 2; - assert(mid < trims.size() and mid >= 0); - std::int64_t trim = trims.rbegin()[mid]; - bool passed = verify_reduced(p, trim, t, options, vo, inputs, tols); - if(passed) - { - left = mid + 1; - } - else - { - failed = trim; - right = mid - 1; - } - } - if(failed > 0) - { - std::cout << "Failure starts at: " << failed << std::endl; - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx diff --git a/docker/rocm/migraphx/driver/verify.hpp b/docker/rocm/migraphx/driver/verify.hpp deleted file mode 100644 index c0a2ff22b..000000000 --- a/docker/rocm/migraphx/driver/verify.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DRIVER_VERIFY_HPP -#define MIGRAPHX_GUARD_RTGLIB_DRIVER_VERIFY_HPP - -#include "verify_options.hpp" -#include -#include - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -verify::tolerance get_tolerances(const program& p, - verify_options vo, - std::optional rms_tol, - std::optional atol, - std::optional rtol); - -bool verify_program(const std::string& name, - const program& p, - const target& t, - compile_options options = compile_options{}, - verify_options vo = verify_options{}, - const parameter_map& inputs = {}, - verify::tolerance tols = verify::tolerance{}); -void verify_instructions(const program& prog, - const target& t, - compile_options options = compile_options{}, - verify_options vo = verify_options{}, - verify::tolerance tols = verify::tolerance{}); -void verify_reduced_program(const program& p, - const target& t, - compile_options options = compile_options{}, - verify_options vo = verify_options{}, - const parameter_map& inputs = {}, - verify::tolerance tols = verify::tolerance{}); -void verify_bisected_program(const program& p, - const target& t, - compile_options options = compile_options{}, - verify_options vo = verify_options{}, - const parameter_map& inputs = {}, - verify::tolerance tols = verify::tolerance{}); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/driver/verify_options.hpp b/docker/rocm/migraphx/driver/verify_options.hpp deleted file mode 100644 index e25d24c08..000000000 --- a/docker/rocm/migraphx/driver/verify_options.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DRIVER_VERIFY_OPTIONS_HPP -#define MIGRAPHX_GUARD_RTGLIB_DRIVER_VERIFY_OPTIONS_HPP - -#include "precision.hpp" - -namespace migraphx { -namespace driver { -inline namespace MIGRAPHX_INLINE_NS { - -struct verify_options -{ - /// Quantization precision - precision quantize = precision::fp32; - - /** - * Converts floating point values to double on the ref target. - */ - bool ref_use_double = false; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace driver -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/dynamic_loader.cpp b/docker/rocm/migraphx/dynamic_loader.cpp deleted file mode 100644 index b78f9e4be..000000000 --- a/docker/rocm/migraphx/dynamic_loader.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -// cppcheck-suppress definePrefix -#define WIN32_LEAN_AND_MEAN -#include -#else -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifndef _WIN32 - -void check_load_error(bool flush = false) -{ - char* error_msg = dlerror(); - if(not flush and error_msg != nullptr) - MIGRAPHX_THROW("Dynamic loading or symbol lookup failed with " + std::string(error_msg)); -} - -struct dynamic_loader_impl -{ - dynamic_loader_impl() = default; - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wignored-attributes" -#endif - dynamic_loader_impl(const fs::path& p, std::shared_ptr t = nullptr) - : handle(dlopen(p.string().c_str(), RTLD_GLOBAL | RTLD_NOW), - manage_deleter{}), - temp(std::move(t)) - { - check_load_error(); - } - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - - static std::shared_ptr from_buffer(const char* image, std::size_t size) - { - auto t = std::make_shared("dloader"); - auto f = t->path / "libtmp.so"; - write_buffer(f, image, size); - return std::make_shared(f, t); - } - - std::shared_ptr handle = nullptr; - std::shared_ptr temp = nullptr; -}; - -fs::path dynamic_loader::path(void* address) -{ - fs::path p; - Dl_info info; - // Find the location of .so - if(dladdr(address, &info) != 0) - p = info.dli_fname; - return p; -} - -#else - -struct dynamic_loader_impl -{ - dynamic_loader_impl() = default; - dynamic_loader_impl(const fs::path& p, tmp_dir t = {}) - : handle{LoadLibrary(p.string().c_str())}, temp{std::move(t)} - { - if(handle == nullptr) - { - MIGRAPHX_THROW("Error loading DLL: " + p.string() + " (" + - std::to_string(GetLastError()) + ")"); - } - } - - dynamic_loader_impl(const dynamic_loader_impl&) = delete; - dynamic_loader_impl& operator=(const dynamic_loader_impl&) = delete; - - dynamic_loader_impl(dynamic_loader_impl&&) = default; - - ~dynamic_loader_impl() - { - if(handle != nullptr) - { - FreeLibrary(handle); - } - } - - static std::shared_ptr from_buffer(const char* image, std::size_t size) - { - auto t = tmp_dir{"migx-dynload"}; - auto f = t.path / "tmp.dll"; - write_buffer(f, image, size); - return std::make_shared(f, std::move(t)); - } - - HMODULE handle = nullptr; - tmp_dir temp; -}; - -fs::path dynamic_loader::path(void* address) -{ - HMODULE module = nullptr; - if(GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - static_cast(address), - &module) == 0) - { - auto err = GetLastError(); - MIGRAPHX_THROW("Unable to obtain module handle, error = " + std::to_string(err)); - } - TCHAR buffer[MAX_PATH]; - if(GetModuleFileName(module, buffer, sizeof(buffer)) == 0) - { - auto err = GetLastError(); - MIGRAPHX_THROW("Unable to read module file path, error = " + std::to_string(err)); - } - if(GetLastError() == ERROR_INSUFFICIENT_BUFFER) - { - MIGRAPHX_THROW("Buffer too small (" + std::to_string(MAX_PATH) + ") to hold the path"); - } - return {buffer}; -} - -#endif - -optional dynamic_loader::try_load(const fs::path& p) -{ - try - { - return dynamic_loader{p}; - } - catch(const std::exception&) - { - return nullopt; - } -} - -dynamic_loader::dynamic_loader(const fs::path& p) : impl(std::make_shared(p)) -{ -} - -dynamic_loader::dynamic_loader(const char* image, std::size_t size) - : impl(dynamic_loader_impl::from_buffer(image, size)) -{ -} - -dynamic_loader::dynamic_loader(const std::vector& buffer) - : impl(dynamic_loader_impl::from_buffer(buffer.data(), buffer.size())) -{ -} - -std::shared_ptr dynamic_loader::get_symbol(const std::string& name) const -{ -#ifndef _WIN32 - // flush any previous error messages - check_load_error(true); - void* symbol = dlsym(impl->handle.get(), name.c_str()); - if(symbol == nullptr) - check_load_error(); - return {impl, symbol}; -#else - FARPROC addr = GetProcAddress(impl->handle, name.c_str()); - if(addr == nullptr) - MIGRAPHX_THROW("Symbol not found: " + name + " (" + std::to_string(GetLastError()) + ")"); - return {impl, reinterpret_cast(addr)}; -#endif -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_allocation.cpp b/docker/rocm/migraphx/eliminate_allocation.cpp deleted file mode 100644 index 1b61d64cb..000000000 --- a/docker/rocm/migraphx/eliminate_allocation.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void eliminate_allocation::apply(module& m) const -{ - assert(alignment > 0); - - std::size_t n = 0; - std::vector> allocs; - for(auto ins : iterator_for(m)) - { - if(ins->name() != allocation_op) - continue; - allocs.emplace_back(ins, n); - std::size_t size = ins->get_shape().bytes(); - std::size_t padding = (alignment - (size % alignment)) % alignment; - n += size + padding; - } - if(n > 0) - { - auto mem = m.add_parameter("memory", shape{shape::int8_type, {n}}); - for(auto&& pp : allocs) - { - auto ins = pp.first; - auto s = ins->get_shape(); - auto offset = pp.second; - m.replace_instruction( - ins, make_op("load", {{"shape", to_value(s)}, {"offset", offset}}), mem); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_common_subexpression.cpp b/docker/rocm/migraphx/eliminate_common_subexpression.cpp deleted file mode 100644 index e5ae2668c..000000000 --- a/docker/rocm/migraphx/eliminate_common_subexpression.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -void cse_range(module& m, Range&& r) -{ - std::unordered_multimap instructions; - std::unordered_set processed_ins; - for(auto ins : r) - { - // Skip dead instructions - if(ins->outputs().empty()) - continue; - - // Find instruction with the same name - auto found_instructions = range(instructions.equal_range(ins->name())); - for(const auto& pp : found_instructions) - { - auto eq = pp.second; - if(contains(processed_ins, eq)) - continue; - if(*eq != *ins) - continue; - m.replace_instruction(ins, eq); - processed_ins.emplace(ins); - std::vector outputs; - std::copy_if(eq->outputs().begin(), - eq->outputs().end(), - std::back_inserter(outputs), - [&](auto x) { return m.has_instruction(x); }); - - std::sort(outputs.begin(), outputs.end(), [&](auto x, auto y) { - return std::distance(eq, x) < std::distance(eq, y); - }); - cse_range(m, outputs); - } - instructions.emplace(ins->name(), ins); - } -} - -void eliminate_common_subexpression::apply(module& m) const { cse_range(m, iterator_for(m)); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_concat.cpp b/docker/rocm/migraphx/eliminate_concat.cpp deleted file mode 100644 index 47a095659..000000000 --- a/docker/rocm/migraphx/eliminate_concat.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -void eliminate_concat::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - auto concat_op = concat_opt.get_concat(ins->get_operator()); - // Look for the concat operator - if(not concat_op.has_value()) - continue; - // If any inputs are builtin or context free then abort - // If any inputs are used more than once, then abort since there could - // be errors due to aliasing - if(std::any_of(ins->inputs().begin(), ins->inputs().end(), [](auto arg) { - return arg->name().front() == '@' or - (arg->get_operator().is_context_free() and - not contains({"concat", "identity"}, arg->name())) or - arg->outputs().size() > 1; - })) - continue; - // We can only do this optimization when concat axis is either the leftmost - // axis OR the sizes to the left of this axis are all equal to 1 - // Since we've already checked that the non-axis dimensions are identical - // we only need to check the first input - auto lens = ins->inputs().front()->get_shape().lens(); - std::size_t axis_index = tune_axis(lens.size(), concat_op->axis, concat_op->name()); - if(axis_index == 0 or - std::all_of(lens.begin(), lens.begin() + axis_index, [](auto x) { return x == 1; })) - { - // Last input should be an allocation - auto last = ins->inputs().back(); - if(last->name() != concat_opt.allocate()) - continue; - // Where are the allocations for the tensors to be concatenated? - std::vector allocations; - - std::transform( - ins->inputs().begin(), - std::prev(ins->inputs().end()), - std::back_inserter(allocations), - [&](instruction_ref x) { return instruction::get_output_alias(x, true); }); - - if(std::any_of(allocations.begin(), allocations.end(), [&](auto x) { - return x->name() != concat_opt.allocate(); - })) - continue; - - // Need to sort the allocations, so that we know where to - // insert the "super"-allocation - auto sorted_allocations = allocations; - std::sort(sorted_allocations.begin(), - sorted_allocations.end(), - [&](instruction_ref x, instruction_ref y) { - return std::distance(m.begin(), x) < std::distance(m.begin(), y); - }); - // Move "super" allocation to the front - auto first = sorted_allocations.front(); - auto super = m.move_instruction(last, first); - // Replace each allocation with a load - std::size_t offset = 0; - for(auto alloc : allocations) - { - op::load op{alloc->get_shape(), offset}; - m.replace_instruction(alloc, op, {super}); - offset += alloc->get_shape().bytes(); - } - std::vector args = {super}; - std::copy(ins->inputs().begin(), ins->inputs().end() - 1, std::back_inserter(args)); - m.replace_instruction(ins, migraphx::make_op("identity"), args); - } - } -} -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_contiguous.cpp b/docker/rocm/migraphx/eliminate_contiguous.cpp deleted file mode 100644 index 02c18ccf8..000000000 --- a/docker/rocm/migraphx/eliminate_contiguous.cpp +++ /dev/null @@ -1,195 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_ELIMINATE_CONTIGUOUS) - -static bool try_compute_shape(instruction_ref ins, - const std::vector& inputs, - const std::vector& mods) -{ - try - { - shape new_shape = ins->get_operator().compute_shape(inputs, mods); - - // Cannot tell if a dynamic shape will need to be made contiguous - if(new_shape.dynamic()) - { - return false; - } - - // If the output shape is a standard shape, no need to try its output - if(new_shape.standard()) - { - return true; - } - - // if no changes for the shape, the contiguous can also be removed - if(new_shape == ins->get_shape()) - { - return true; - } - - auto outputs = ins->outputs(); - // If the current instruction has no output, it means it is the last - // instruction and generates a non-standard output shape, and the last - // output shape is different from the case with the contiguous operator - if(outputs.empty()) - { - return false; - } - - for(auto output : outputs) - { - auto args = output->inputs(); - std::vector input_shapes(args.size()); - std::transform(args.begin(), args.end(), input_shapes.begin(), [&](auto& arg) { - return (arg == ins) ? new_shape : arg->get_shape(); - }); - - if(not try_compute_shape(output, input_shapes, output->module_inputs())) - { - return false; - } - } - } - catch(const std::exception& e) - { - if(enabled(MIGRAPHX_TRACE_ELIMINATE_CONTIGUOUS{})) - { - std::cout << "Exception: " << e.what() << std::endl; - } - return false; - } - catch(...) - { - if(enabled(MIGRAPHX_TRACE_ELIMINATE_CONTIGUOUS{})) - { - std::cout << "Unknown exception" << std::endl; - } - return false; - } - - return true; -} - -static bool try_compute_shape(instruction_ref ins, - const std::vector& args, - const std::vector& mods) -{ - auto inputs = to_shapes(args); - return try_compute_shape(ins, inputs, mods); -} - -template -static void remove_contiguous(const std::string& op_name, module& m, F f) -{ - auto last = std::prev(m.end()); - std::vector const_instructions; - - for(auto ins : iterator_for(m)) - { - // return instruction should have inputs with standard shape - if(ins->name() == "@return") - continue; - - if(ins != last and ins->outputs().empty()) - continue; - - if(not f(ins)) - continue; - - auto args = ins->inputs(); - auto mod_args = ins->module_inputs(); - - for(auto arg : ins->inputs()) - { - if(arg->name() != op_name) - continue; - if(enabled(MIGRAPHX_TRACE_ELIMINATE_CONTIGUOUS{})) - { - std::cout << "eliminate_contiguous: "; - m.debug_print(ins); - } - auto prev = arg->inputs().front(); - // create copy of args each time as they are modified inside the loop - auto new_args = ins->inputs(); - replace(new_args, arg, prev); - if(try_compute_shape(ins, new_args, mod_args)) - { - instruction::replace_argument(ins, arg, prev); - } - else if(prev->can_eval()) - { - const_instructions.push_back(arg); - } - } - } - - // Perform static contiguous evaluations in parallel - std::vector literals(const_instructions.size()); - par_for(const_instructions.size(), 1, [&](const auto i) { - auto c = op::contiguous{}; - auto prev = const_instructions[i]->inputs().front(); - // compute the output contiguous shape from the previous instruction shape - shape computed_shape = c.compute_shape({prev->get_shape()}); - const std::vector& prev_eval = {prev->eval()}; - // prev_eval should not be used in make_compute_output_shape() as computed_shape is static - auto co_shape = make_compute_output_shape(pack(c, computed_shape, prev_eval)); - literals[i] = c.compute(co_shape, prev_eval); - }); - - // Replace static contiguous operations with a literal - for(size_t i = 0; i < const_instructions.size(); i++) - { - auto l = m.add_literal(literals[i].get_shape(), literals[i].data()); - m.replace_instruction(const_instructions[i], l); - } -} - -void eliminate_contiguous::apply(module& m) const -{ - // Skip contiguous from splits first - remove_contiguous(op_name, m, [](auto ins) { - if(ins->name() != "slice") - return true; - return (ins->inputs().front()->outputs().size() == 1); - }); - remove_contiguous(op_name, m, [](auto) { return true; }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_convert.cpp b/docker/rocm/migraphx/eliminate_convert.cpp deleted file mode 100644 index cd96be349..000000000 --- a/docker/rocm/migraphx/eliminate_convert.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * The MIT License (MIT) - ,* - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Matches with some sequence of sequential convert instructions. - * If input to the sequence of converts has the same shape as the last convert, - * replace last convert with the input. - * If input to the sequence is not the same shape as the last convert, - * replace last convert with convert from the input to the last shape. - */ -struct find_nested_convert -{ - auto matcher() const { return match::name("convert")(match::arg(0)(match::name("convert"))); } - - void apply(module& m, const match::matcher_result& mr) const - { - auto matched_ins = mr.result; - auto prev_convert = matched_ins->inputs().front(); - auto input = prev_convert->inputs().front(); - while(input->name() == "convert") - { - input = input->inputs().front(); - } - if(matched_ins->get_shape() == input->get_shape()) - { - m.replace_instruction(matched_ins, input); - } - else - { - m.replace_instruction(matched_ins, matched_ins->get_operator(), input); - } - } -}; - -struct find_nop_converts -{ - auto matcher() const { return match::name("convert")(match::same_shape(match::arg(0))); } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - m.replace_instruction(ins, ins->inputs().front()); - } -}; - -void eliminate_convert::apply(module& m) const -{ - match::find_matches(m, find_nested_convert{}, find_nop_converts{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_data_type.cpp b/docker/rocm/migraphx/eliminate_data_type.cpp deleted file mode 100644 index fc8144489..000000000 --- a/docker/rocm/migraphx/eliminate_data_type.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void insert_convert_to_supported_type(module& m, - instruction_ref ins, - migraphx::shape::type_t target_type, - std::set unsupported_types) -{ - migraphx::shape::type_t orig_type = ins->get_shape().type(); - std::vector inputs = ins->inputs(); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](const auto& i) { - if(contains(unsupported_types, i->get_shape().type())) - { - return m.insert_instruction( - ins, - migraphx::make_op("convert", {{"target_type", migraphx::to_value(target_type)}}), - i); - } - else - { - return i; - } - }); - // if no change - if(inputs == ins->inputs()) - return; - auto op = ins->get_operator(); - auto attributes = op.attributes(); - if(attributes.contains("general_data_type")) - { - op = make_op(attributes["general_data_type"].to(), op.to_value()); - } - auto new_ins = m.insert_instruction(ins, op, inputs); - if(orig_type == shape::tuple_type) - { - auto orig_outs = ins->outputs(); - if(not std::all_of(orig_outs.begin(), orig_outs.end(), [&](const auto out_ins) { - return out_ins->name() == "get_tuple_elem"; - })) - MIGRAPHX_THROW( - "eliminate_data_type: Instruction with tuple output doesn't have all its " - "usages as get_tuple_elem instruction"); - - std::transform( - orig_outs.begin(), orig_outs.end(), orig_outs.begin(), [&](const auto out_ins) { - auto gte_ins = m.insert_instruction(ins, out_ins->get_operator(), new_ins); - auto orig_out_type = out_ins->get_shape().type(); - if(contains(unsupported_types, orig_out_type)) - { - auto gte_convert = m.insert_instruction( - ins, make_op("convert", {{"target_type", orig_out_type}}), gte_ins); - return m.replace_instruction(out_ins, gte_convert); - } - else - { - return m.replace_instruction(out_ins, gte_ins); - } - }); - } - else - { - auto convert_back_ins = m.insert_instruction( - ins, - migraphx::make_op("convert", {{"target_type", migraphx::to_value(orig_type)}}), - new_ins); - m.replace_instruction(ins, convert_back_ins); - } -} - -void eliminate_data_type::apply(module& m) const -{ - static const std::vector skip_op_names = {"convert", - "get_tuple_elem", - "if", - "loop", - "roialign", - "nonmaxsuppression", - "scatternd_add", - "scatternd_mul", - "scatternd_none", - "select_module"}; - if(unsupported_types.empty()) - return; - - for(auto ins : iterator_for(m)) - { - if(ins->name()[0] == '@') - continue; - if(contains(skip_op_names, ins->name()) and not contains(unsupported_ops, ins->name())) - continue; - if(contains(unsupported_ops, "all") or contains(unsupported_ops, ins->name())) - insert_convert_to_supported_type(m, ins, target_type, unsupported_types); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_identity.cpp b/docker/rocm/migraphx/eliminate_identity.cpp deleted file mode 100644 index 1a914d6cc..000000000 --- a/docker/rocm/migraphx/eliminate_identity.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void eliminate_identity::apply(module& m) const -{ - auto last = std::prev(m.end()); - for(auto ins : iterator_for(m)) - { - // Skip the first instruction, since we always process the previous - // instruction - if(ins == m.begin()) - continue; - const auto i = std::prev(ins); - - if(i->name() == "identity") - { - m.replace_instruction(i, i->inputs().front()); - m.move_instruction(i, m.end()); - } - if(ins == last) - { - if(ins->name() == "identity") - { - const instruction_ref& identity_input = ins->inputs().front(); - if(identity_input->outputs().size() == 1) - { - m.move_instruction(identity_input, last); - // since this is the last instruction, removing it only - // requires changing "last" and calling remove below - last = std::prev(last); - } - } - break; - } - } - m.remove_instructions(std::next(last), m.end()); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/eliminate_pad.cpp b/docker/rocm/migraphx/eliminate_pad.cpp deleted file mode 100644 index b6fb08bc8..000000000 --- a/docker/rocm/migraphx/eliminate_pad.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static void update_op(const instruction_ref& input, const instruction_ref& ins, module& m) -{ - auto pad_op = any_cast(input->get_operator()); - - auto kdims = input->get_shape().lens().size() - 2; - auto kdims_it = pad_op.pads.begin() + 2; - - std::vector pads_l(kdims_it, kdims_it + kdims); - std::vector pads_r(kdims_it + kdims + 2, pad_op.pads.end()); - - auto op = ins->get_operator(); - std::vector padding(kdims * 2, 0); - - std::transform( - pads_l.begin(), pads_l.end(), padding.begin(), padding.begin(), std::plus()); - std::transform(pads_r.begin(), - pads_r.end(), - padding.begin() + kdims, - padding.begin() + kdims, - std::plus()); - - op.from_value({{"padding", padding}}); - - std::vector new_inputs{ins->inputs()}; - new_inputs.front() = input->inputs().front(); - - m.replace_instruction(ins, op, new_inputs); -} - -static void update_pooling(const instruction_ref& input, const instruction_ref& ins, module& m) -{ - auto op = any_cast(ins->get_operator()); - if(op.mode == op::pooling_mode::average) - { - return; - } - auto pad_op = any_cast(input->get_operator()); - - auto kdims = input->get_shape().lens().size() - 2; - auto kdims_it = pad_op.pads.begin() + 2; - - std::vector pads_l(kdims_it, kdims_it + kdims); - std::vector pads_r(kdims_it + kdims + 2, pad_op.pads.end()); - - std::transform( - pads_l.begin(), pads_l.end(), op.padding.begin(), op.padding.begin(), std::plus()); - std::transform(pads_r.begin(), - pads_r.end(), - op.padding.begin() + kdims, - op.padding.begin() + kdims, - std::plus()); - - std::vector new_inputs{ins->inputs()}; - new_inputs.front() = input->inputs().front(); - - m.replace_instruction(ins, op, new_inputs); -} - -void eliminate_pad::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - const std::string& op_name = ins->name(); - if(op_name != "convolution" and op_name != "im2col" and op_name != "pooling") - continue; - auto input = ins->inputs().front(); - if(input->name() != "pad") - continue; - if(op_name == "convolution" or op_name == "im2col") - update_op(input, ins, m); - else if(op_name == "pooling") - update_pooling(input, ins, m); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/env.cpp b/docker/rocm/migraphx/env.cpp deleted file mode 100644 index 454d50810..000000000 --- a/docker/rocm/migraphx/env.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -bool enabled(const char* name) -{ - auto e = env(name); - if(e.empty()) - return false; - return contains({"1", "enable", "enabled", "yes", "true"}, e.front()); -} - -bool disabled(const char* name) -{ - auto e = env(name); - if(e.empty()) - return false; - return contains({"0", "disable", "disabled", "no", "false"}, e.front()); -} - -std::size_t value_of(const char* name, std::size_t fallback) -{ - auto e = env(name); - if(e.empty()) - return fallback; - return std::stoul(e.front()); -} - -std::string string_value_of(const char* name, std::string fallback) -{ - auto e = env(name); - if(e.empty()) - return fallback; - return e.front(); -} - -std::vector env(const char* name) -{ - auto* p = std::getenv(name); - if(p == nullptr) - return {}; - else - return {{p}}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/file_buffer.cpp b/docker/rocm/migraphx/file_buffer.cpp deleted file mode 100644 index dabd1e882..000000000 --- a/docker/rocm/migraphx/file_buffer.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -T generic_read_file(const fs::path& filename, size_t offset = 0, size_t nbytes = 0) -{ - std::ifstream is(filename, std::ios::binary | std::ios::ate); - if(not is.is_open()) - MIGRAPHX_THROW("Failure opening file: " + filename); - if(nbytes == 0) - { - // if there is a non-zero offset and nbytes is not set, - // calculate size of remaining bytes to read - nbytes = is.tellg(); - if(offset > nbytes) - MIGRAPHX_THROW("offset is larger than file size"); - nbytes -= offset; - } - if(nbytes < 1) - MIGRAPHX_THROW("Invalid size for: " + filename); - is.seekg(offset, std::ios::beg); - - T buffer(nbytes, 0); - if(not is.read(&buffer[0], nbytes)) - MIGRAPHX_THROW("Error reading file: " + filename); - return buffer; -} - -std::vector read_buffer(const fs::path& filename, size_t offset, size_t nbytes) -{ - return generic_read_file>(filename, offset, nbytes); -} - -std::string read_string(const fs::path& filename) -{ - return generic_read_file(filename); -} - -void write_string(const fs::path& filename, const std::string& buffer) -{ - write_buffer(filename, buffer.data(), buffer.size()); -} - -void write_buffer(const fs::path& filename, const char* buffer, std::size_t size) -{ - std::ofstream os(filename, std::ios::out | std::ios::binary); - os.write(buffer, size); -} - -void write_buffer(const fs::path& filename, const std::vector& buffer) -{ - write_buffer(filename, buffer.data(), buffer.size()); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fileutils.cpp b/docker/rocm/migraphx/fileutils.cpp deleted file mode 100644 index 402e0113e..000000000 --- a/docker/rocm/migraphx/fileutils.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef _WIN32 -constexpr std::string_view executable_postfix{".exe"}; -constexpr std::string_view library_prefix{""}; -constexpr std::string_view library_postfix{".dll"}; -constexpr std::string_view static_library_postfix{".lib"}; -constexpr std::string_view object_file_postfix{".obj"}; -#else -constexpr std::string_view executable_postfix{""}; -constexpr std::string_view library_prefix{"lib"}; -constexpr std::string_view library_postfix{".so"}; -constexpr std::string_view static_library_postfix{".a"}; -constexpr std::string_view object_file_postfix{".o"}; -#endif - -fs::path make_executable_filename(std::string_view name) -{ - return std::string{name}.append(executable_postfix); -} - -fs::path make_shared_object_filename(std::string_view name) -{ - return std::string{library_prefix}.append(name).append(library_postfix); -} - -fs::path make_object_file_filename(std::string_view name) -{ - return std::string{name}.append(object_file_postfix); -} - -fs::path make_static_library_filename(std::string_view name) -{ - return std::string{library_prefix}.append(name).append(static_library_postfix); -} - -fs::path append_extension(const fs::path& path, std::string_view ext) -{ - return fs::path{path}.replace_extension(path.extension().string().append(ext)); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fp8_ocp_to_fnuz.cpp b/docker/rocm/migraphx/fp8_ocp_to_fnuz.cpp deleted file mode 100644 index 305ca6058..000000000 --- a/docker/rocm/migraphx/fp8_ocp_to_fnuz.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace { - -using fp8::fp8e4m3fnuz; - -std::unordered_set get_quantizable_op_names() -{ - static std::unordered_set s = {"convolution", "dot"}; - return s; -} - -struct match_fp8ocp_convert_to_fp8fnuz -{ - auto matcher() const - { - auto dq1 = match::arg(0)( - skip_post_dq_ops(match::dequantizelinear_op("scale1", "zp1").bind("dq1"))); - auto dq2 = match::arg(1)( - skip_post_dq_ops(match::dequantizelinear_op("scale2", "zp2").bind("dq2"))); - return match::name(get_quantizable_op_names())(dq1, dq2); - } - - static auto bit_cast_and_handle_specials(module& m, - const instruction_ref dq, - const instruction_ref x, - const instruction_ref bits_0x80_lit, - const instruction_ref bits_0x7f_lit, - const instruction_ref bits_0xff_lit, - const instruction_ref bits_0x00_lit) - { - auto x_lens = x->get_shape().lens(); - auto cast_input = m.insert_instruction( - dq, make_op("bit_cast", {{"target_type", shape::fp8e4m3fnuz_type}}), x); - auto mb_bits_0x80_lit = m.insert_instruction( - dq, make_op("multibroadcast", {{"out_lens", x_lens}}), bits_0x80_lit); - auto mb_bits_0x7f_lit = m.insert_instruction( - dq, make_op("multibroadcast", {{"out_lens", x_lens}}), bits_0x7f_lit); - auto mb_bits_0xff_lit = m.insert_instruction( - dq, make_op("multibroadcast", {{"out_lens", x_lens}}), bits_0xff_lit); - auto mb_zero_lit = m.insert_instruction( - dq, make_op("multibroadcast", {{"out_lens", x_lens}}), bits_0x00_lit); - // negative zero in fp8e4m3fn to zero in fp8e4m3fnuz - // a == 0x80 ? 0x0 : a - auto is_neg_zero = m.insert_instruction(dq, make_op("equal"), cast_input, mb_bits_0x80_lit); - auto ret = m.insert_instruction(dq, make_op("where"), is_neg_zero, mb_zero_lit, cast_input); - - // positive and negative NaN in fp8e4m3fn to NaN in fp8e4m3fnuz - // (a == 0x7f or a == 0xff) ? 0x80 : a - auto eq_0x7f = m.insert_instruction(dq, make_op("equal"), ret, mb_bits_0x7f_lit); - - auto eq_0xff = m.insert_instruction(dq, make_op("equal"), ret, mb_bits_0xff_lit); - - auto cond = m.insert_instruction(dq, make_op("logical_or"), eq_0x7f, eq_0xff); - ret = m.insert_instruction(dq, make_op("where"), cond, mb_bits_0x80_lit, ret); - return ret; - } - - // Add the same broadcast instructions after adjusted scales or - // adjusted zero points from after the originals. Similar to - // propagate_quantized_ins in simplify_qdq. - static auto propagate_broadcasts(module& m, - const instruction_ref adj, - const instruction_ref ori, - const instruction_ref start, - const instruction_ref insert_pt) - { - auto prev_ins = start; - std::vector ins_between; - // matcher skips continguous, multi/broadcasts and transposes, collect all those - // instructions - while(prev_ins != ori) - { - ins_between.push_back(prev_ins); - prev_ins = prev_ins->inputs().front(); - } - auto ret = adj; - for(auto ins : reverse_iterator_for(ins_between)) - { - ret = m.insert_instruction(insert_pt, (*ins)->get_operator(), {ret}); - } - return ret; - } - - static auto cast_to_fnuz(module& m, - const instruction_ref dq, - const instruction_ref input, - const instruction_ref dq_scale, - const instruction_ref dq_zp) - { - auto x = input; - std::vector bits_0x80 = {fp8e4m3fnuz(0x80, fp8e4m3fnuz::from_bits())}; - auto bits_0x80_lit = m.add_literal(shape{shape::fp8e4m3fnuz_type, {1}, {0}}, bits_0x80); - - std::vector bits_0x7f = {fp8e4m3fnuz(0x7f, fp8e4m3fnuz::from_bits())}; - auto bits_0x7f_lit = m.add_literal(shape{shape::fp8e4m3fnuz_type, {1}, {0}}, bits_0x7f); - - std::vector bits_0xff = {fp8e4m3fnuz(0xff, fp8e4m3fnuz::from_bits())}; - auto bits_0xff_lit = m.add_literal(shape{shape::fp8e4m3fnuz_type, {1}, {0}}, bits_0xff); - - std::vector bits_0x00 = {fp8e4m3fnuz(0x00, fp8e4m3fnuz::from_bits())}; - auto bits_0x00_lit = m.add_literal(shape{shape::fp8e4m3fnuz_type, {1}, {0}}, bits_0x00); - - x = bit_cast_and_handle_specials( - m, dq, x, bits_0x80_lit, bits_0x7f_lit, bits_0xff_lit, bits_0x00_lit); - auto adj_dq_zp = bit_cast_and_handle_specials( - m, dq, dq_zp, bits_0x80_lit, bits_0x7f_lit, bits_0xff_lit, bits_0x00_lit); - - // adj_scale = 2 * scale - auto two_lit = m.add_literal(literal{shape{dq_scale->get_shape().type()}, {2}}); - two_lit = m.insert_instruction( - dq, make_op("multibroadcast", {{"out_lens", dq_scale->get_shape().lens()}}), two_lit); - auto adj_dq_scale = m.insert_instruction(dq, make_op("mul"), dq_scale, two_lit); - - adj_dq_scale = propagate_broadcasts(m, adj_dq_scale, dq_scale, dq->inputs().at(1), dq); - adj_dq_zp = propagate_broadcasts(m, adj_dq_zp, dq_zp, dq->inputs().at(2), dq); - m.replace_instruction(dq, make_op("dequantizelinear"), x, adj_dq_scale, adj_dq_zp); - } - - auto apply(module& m, const match::matcher_result& r) const - { - auto dq1 = r.instructions["dq1"]; - auto dq2 = r.instructions["dq2"]; - auto scale1 = r.instructions["scale1"]; - auto scale2 = r.instructions["scale2"]; - auto zp1 = r.instructions["zp1"]; - auto zp2 = r.instructions["zp2"]; - - std::set supported_types = {migraphx::shape::fp8e4m3fn_type}; - if(not contains(supported_types, dq1->inputs().front()->get_shape().type()) or - not contains(supported_types, dq2->inputs().front()->get_shape().type())) - return; - - cast_to_fnuz(m, dq1, dq1->inputs().front(), scale1, zp1); - cast_to_fnuz(m, dq2, dq2->inputs().front(), scale2, zp2); - } -}; - -} // namespace - -void fp8_ocp_to_fnuz::apply(module_pass_manager& mpm) const -{ - module_ref mm = &mpm.get_module(); - match::find_matches(*mm, match_fp8ocp_convert_to_fp8fnuz{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fp_to_double.cpp b/docker/rocm/migraphx/fp_to_double.cpp deleted file mode 100644 index 34cded8ce..000000000 --- a/docker/rocm/migraphx/fp_to_double.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void fp_to_double::apply(module_pass_manager& mpm) const -{ - mpm.run_pass(eliminate_data_type{convert_fp_types, shape::type_t::double_type}); - mpm.run_pass(eliminate_convert{}); - mpm.run_pass(migraphx::dead_code_elimination{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fuse_concat.cpp b/docker/rocm/migraphx/fuse_concat.cpp deleted file mode 100644 index 1bc398833..000000000 --- a/docker/rocm/migraphx/fuse_concat.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -unsigned int get_noop_counter() -{ - static unsigned int counter = 0; - return counter++; -} - -struct fused_concat -{ - int64_t axis = 0; - - std::string name() const { return "fused_concat"; } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - shape compute_shape(std::vector inputs, const std::vector& mods) const - { - check_shapes{inputs, *this}.same_ndims(); - // original concat can have multiple inputs. Let's say it has `n` input args. - // Each of those `n` input args are converted into pointwise modules that take atleast 1 - // input parameter. Fused concat will have `n+1` module arguments. `n+1`th module is the - // post pointwise module which can take 0 or more input arguments. - if((inputs.size() + 1) < mods.size()) - MIGRAPHX_THROW("FUSED_CONCAT: Missing fused modules inputs parameters"); - auto input_iter = inputs.begin(); - std::vector concat_inputs; - for(module_ref mod : range(mods.begin(), mods.end() - 1)) - { - concat_inputs.push_back(*input_iter); - input_iter += mod->get_parameter_names().size(); - } - module_ref post_mod = mods.back(); - // post_mod has one input argument that is result of concat and will get generated from - // pre-mods internally. Therefore deduct 1 from post_mod params while asserting. - assert(input_iter + post_mod->get_parameter_names().size() - 1 == inputs.end()); - auto type = std::prev(post_mod->end())->get_shape().type(); - const auto& first_shape_lens = concat_inputs.front().lens(); - auto mismatch_it = - std::find_if_not(concat_inputs.begin() + 1, concat_inputs.end(), [&](auto s) { - const auto& lens = s.lens(); - return std::equal(lens.begin(), - lens.begin() + axis, - first_shape_lens.begin(), - first_shape_lens.begin() + axis) and - std::equal(lens.begin() + axis + 1, - lens.end(), - first_shape_lens.begin() + axis + 1, - first_shape_lens.end()); - }); - if(mismatch_it != concat_inputs.end()) - MIGRAPHX_THROW("FUSED_CONCAT: all input dimensions should match along non-axis of " + - std::to_string(axis) + ": {" + to_string_range(first_shape_lens) + - "} != {" + to_string_range(mismatch_it->lens()) + "}"); - - std::size_t new_dim_axis = transform_accumulate( - concat_inputs.begin(), concat_inputs.end(), 0, std::plus<>{}, [&](const auto& input) { - return input.lens()[axis]; - }); - auto new_lens = concat_inputs.front().lens(); - new_lens[axis] = new_dim_axis; - return shape::from_permutation(type, new_lens, find_permutation(inputs)); - } -}; -MIGRAPHX_REGISTER_OP(fused_concat); - -namespace { -struct find_concat_pointwise -{ - auto matcher() const - { - auto pointwise_used_once = match::name("pointwise")(match::used_once()); - return match::name("concat")(match::used_once(), - match::any_of[match::inputs()](pointwise_used_once)); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto concat_ins = r.result; - - std::vector inputs; - size_t num_noops = 0; - for(auto input : concat_ins->inputs()) - { - if(input->name() == "pointwise" and input->outputs().size() == 1) - { - inputs.insert(inputs.end(), input->inputs().begin(), input->inputs().end()); - } - else - { - num_noops++; - inputs.push_back(input); - } - } - if(num_noops > std::max(size_t{1}, concat_ins->inputs().size() / 4)) - { - return; - } - std::vector module_inputs; - std::transform(concat_ins->inputs().begin(), - concat_ins->inputs().end(), - std::back_inserter(module_inputs), - [&](instruction_ref input) { - if(input->name() == "pointwise" and input->outputs().size() == 1) - { - auto* pm = input->module_inputs().front(); - return mpm.create_module("concat:" + pm->name(), *pm); - } - auto* pm = mpm.create_module("concat:noop" + - std::to_string(get_noop_counter())); - auto x = pm->add_parameter("x0", shape{input->get_shape().type()}); - pm->add_return({x}); - return pm; - }); - auto* post_pm = mpm.create_module("noop:concat" + std::to_string(get_noop_counter())); - auto x = post_pm->add_parameter("!x0", shape{concat_ins->get_shape().type()}); - post_pm->add_return({x}); - module_inputs.push_back(post_pm); - mpm.get_module().replace_instruction( - concat_ins, - make_op("fused_concat", concat_ins->normalized_operator().to_value()), - inputs, - module_inputs); - } -}; - -struct find_pointwise_concat_pointwise -{ - auto matcher() const - { - auto pointwise = match::name("pointwise")(match::used_once()); - auto concat = - match::name("concat")(match::used_once(), match::any_of[match::inputs()](pointwise)); - return match::name("pointwise")(match::any_of[match::inputs()](concat.bind("concat"))); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto concat_ins = r.instructions["concat"]; - - auto concat_arg = std::find(ins->inputs().begin(), ins->inputs().end(), concat_ins) - - ins->inputs().begin(); - std::vector inputs; - for(auto input : concat_ins->inputs()) - { - if(input->name() == "pointwise" and input->outputs().size() == 1) - inputs.insert(inputs.end(), input->inputs().begin(), input->inputs().end()); - else - inputs.push_back(input); - } - std::copy_if(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(inputs), - [&](auto input) { return input != concat_ins; }); - - std::vector module_inputs; - std::transform(concat_ins->inputs().begin(), - concat_ins->inputs().end(), - std::back_inserter(module_inputs), - [&](instruction_ref input) { - if(input->name() == "pointwise" and input->outputs().size() == 1) - { - auto* pm = input->module_inputs().front(); - return mpm.create_module("concat:" + pm->name(), *pm); - } - auto* pm = mpm.create_module("concat:noop" + - std::to_string(get_noop_counter())); - auto x = pm->add_parameter("x0", shape{input->get_shape().type()}); - pm->add_return({x}); - return pm; - }); - - auto* post_pm = ins->module_inputs().front(); - auto* rm = mpm.create_module(post_pm->name() + ":concat", *post_pm); - std::vector names = rm->get_parameter_names(); - std::sort(names.begin(), names.end()); - auto concat_param_name = names[concat_arg]; - auto concat_param = rm->get_parameter(concat_param_name); - auto param = rm->add_parameter("!" + concat_param_name, concat_param->get_shape()); - rm->replace_instruction(concat_param, param); - rm->remove_instruction(concat_param); - - module_inputs.push_back(rm); - mpm.get_module().replace_instruction( - ins, - make_op("fused_concat", concat_ins->normalized_operator().to_value()), - inputs, - module_inputs); - } -}; - -} // namespace - -void fuse_concat::apply(module_pass_manager& mpm) const -{ - match::find_matches(mpm, find_pointwise_concat_pointwise{}); - mpm.run_pass(migraphx::dead_code_elimination{}); - match::find_matches(mpm, find_concat_pointwise{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fuse_pointwise.cpp b/docker/rocm/migraphx/fuse_pointwise.cpp deleted file mode 100644 index 60befcfd4..000000000 --- a/docker/rocm/migraphx/fuse_pointwise.cpp +++ /dev/null @@ -1,262 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_POINTWISE_FUSION) - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static literal get_scalar(instruction_ref ins) -{ - if(contains({"contiguous", "broadcast", "multibroadcast"}, ins->name())) - return get_scalar(ins->inputs().front()); - const auto& s = ins->get_shape(); - if(s.elements() != 1 and not(s.scalar())) - return {}; - if(not ins->can_eval()) - return {}; - auto e = ins->eval(); - literal r{}; - // needed for bool as visit_at invokes as() which promotes bool to int8 - // Without this we'll break type checks for logical ops that are fused. - if(e.get_shape().type() == shape::bool_type) - { - r = literal{e.at()}; - } - else - { - e.visit_at([&](auto x) { r = literal{x}; }); - } - return r; -} - -static void create_pointwise_modules(module_pass_manager& mpm) -{ - std::size_t n = 0; - for(auto ins : iterator_for(mpm.get_module())) - { - if(not ins->get_operator().attributes().get("pointwise", false)) - continue; - if(ins->get_operator().name() == "layout") - continue; - auto* pm = mpm.create_module(mpm.get_module().name() + ":pointwise" + std::to_string(n++)); - pm->set_bypass(); - - std::unordered_map param_map; - std::vector pointwise_inputs; - std::size_t i = 0; - - for(auto input : ins->inputs()) - { - if(contains(param_map, input)) - continue; - auto scalar = get_scalar(input); - if(scalar.empty()) - { - pointwise_inputs.push_back(input); - param_map[input] = - pm->add_parameter(param_name(i), shape{input->get_shape().type()}); - i++; - } - else - { - param_map[input] = pm->add_literal(scalar); - } - } - - // Don't create pointwise module if no inputs are detected - if(pointwise_inputs.empty()) - continue; - - std::vector inputs; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(inputs), - [&](auto input) { return param_map[input]; }); - auto r = pm->add_instruction(ins->get_operator(), inputs); - pm->add_return({r}); - - mpm.get_module().replace_instruction(ins, make_op("pointwise"), pointwise_inputs, {pm}); - } -} - -static module::with_inputs append_pointwise_module(instruction_ref ins, instruction_ref output) -{ - assert(contains(output->inputs(), ins)); - module pm = *ins->module_inputs().at(0); - module_ref xm = output->module_inputs().at(0); - - auto last = std::prev(pm.end()); - assert(last->name() == "@return"); - assert(last->inputs().size() == 1); - - assert(pm.get_parameter_names().size() == ins->inputs().size()); - assert(xm->get_parameter_names().size() == output->inputs().size()); - - std::vector inputs = ins->inputs(); - std::unordered_map map_ins; - std::unordered_map input_map; - // Copy inputs to input_map - for(auto i : range(inputs.size())) - { - auto input = inputs[i]; - auto param = pm.get_parameter(param_name(i)); - assert(param != pm.end()); - input_map[input] = param; - } - // Add the new parameter and additional inputs - for(auto i : range(output->inputs().size())) - { - auto input = output->inputs()[i]; - auto param = xm->get_parameter(param_name(i)); - assert(param != xm->end()); - if(input == ins) - { - map_ins[param] = last->inputs().front(); - input_map[input] = map_ins[param]; - } - // Avoid duplicate paramter inputs - else if(contains(input_map, input)) - { - map_ins[param] = input_map[input]; - } - else - { - map_ins[param] = - pm.add_parameter(param_name(inputs.size()), {input->get_shape().type()}); - inputs.push_back(input); - input_map[input] = map_ins[param]; - } - } - pm.replace_return(pm.insert_instructions(last, xm, &map_ins)); - return {std::move(pm), inputs}; -} - -static bool find_pointwise_modules(module_pass_manager& mpm) -{ - bool changed = false; - auto last = std::prev(mpm.get_module().end()); - for(auto ins : iterator_for(mpm.get_module())) - { - if(ins->name() != "pointwise") - continue; - if(ins->outputs().empty() and ins != last) - continue; - auto it = std::find_if(ins->inputs().begin(), ins->inputs().end(), [&](auto i) { - return i->name() == "pointwise" and i->outputs().size() == 1; - }); - if(it == ins->inputs().end()) - continue; - auto input = *it; - - auto fused = append_pointwise_module(input, ins); - auto name = fused.mod.name(); - mpm.rename_module(name, name + ":" + ins->module_inputs().front()->name() + "-deleted"); - auto* new_pm = mpm.create_module(name, std::move(fused.mod)); - mpm.get_module().replace_instruction(ins, input->get_operator(), fused.inputs, {new_pm}); - - changed = true; - } - return changed; -} - -namespace { -struct pointwise_reshape : rewrite_reshapes_base -{ - static std::string name() { return "pointwise"; } -}; - -struct pointwise_broadcast_pointwise -{ - auto matcher() const - { - auto broadcast_pointwise = - match::name("multibroadcast")( - match::used_once(), - match::args(match::name("pointwise")(match::used_once()).bind("x"))) - .bind("broadcast"); - return match::name("pointwise")(match::any_of[match::inputs()](broadcast_pointwise)); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto broadcast_ins = r.instructions["broadcast"]; - auto x_ins = r.instructions["x"]; - - auto broadcast = broadcast_ins->get_operator(); - - auto x_inputs = x_ins->inputs(); - std::transform(x_inputs.begin(), x_inputs.end(), x_inputs.begin(), [&](auto input) { - return m.insert_instruction(broadcast_ins, broadcast, input); - }); - - m.replace_instruction( - broadcast_ins, x_ins->get_operator(), x_inputs, x_ins->module_inputs()); - } -}; - -} // namespace - -static void rewrite_broadcasts(module_pass_manager& mpm) -{ - match::find_matches(mpm.get_module(), pointwise_broadcast_pointwise{}); - mpm.run_pass(dead_code_elimination{}); -} - -void fuse_pointwise::apply(module_pass_manager& mpm) const -{ - mpm.run_pass(eliminate_identity{}); - create_pointwise_modules(mpm); - mpm.run_pass(dead_code_elimination{}); - if(enabled(MIGRAPHX_DISABLE_POINTWISE_FUSION{})) - { - return; - } - for(int i = 0; i < 8; i++) - { - if(enable_rewrite_reshapes) - mpm.run_pass(rewrite_reshapes{}); - if(enable_rewrite_broadcasts) - rewrite_broadcasts(mpm); - if(not find_pointwise_modules(mpm)) - break; - mpm.run_pass(dead_code_elimination{}); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fuse_pointwise_reduce.cpp b/docker/rocm/migraphx/fuse_pointwise_reduce.cpp deleted file mode 100644 index eb2feb565..000000000 --- a/docker/rocm/migraphx/fuse_pointwise_reduce.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_SPLIT_REDUCE_SIZE); - -static std::size_t get_split_size(std::size_t default_split) -{ - std::string value = string_value_of(MIGRAPHX_SPLIT_REDUCE_SIZE{}); - if(value.empty()) - return default_split; - return std::stoul(value); -} - -void fuse_pointwise_reduce::apply(module_pass_manager& mpm) const -{ - mpm.run_pass(fuse_pointwise{.enable_rewrite_reshapes = false}); - mpm.run_pass(fuse_reduce{.enable_rewrite_reshapes = false}); - mpm.run_pass(fuse_pointwise{.enable_rewrite_reshapes = true}); - mpm.run_pass(fuse_reduce{.enable_rewrite_reshapes = true}); - mpm.run_pass(split_reduce{.split_size = get_split_size(split_size)}); - mpm.run_pass(fuse_pointwise{.enable_rewrite_broadcasts = true}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/fuse_reduce.cpp b/docker/rocm/migraphx/fuse_reduce.cpp deleted file mode 100644 index 5be8fd4a2..000000000 --- a/docker/rocm/migraphx/fuse_reduce.cpp +++ /dev/null @@ -1,440 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_REDUCE_FUSION) - -struct fused_reduce -{ - std::vector axes{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes")); - } - - shape compute_shape(const std::vector& inputs, std::vector mods) const - { - if(mods.size() != 1) - MIGRAPHX_THROW("should have one submodule."); - const auto* sm = mods.front(); - if(sm->get_output_shapes().size() != 1) - MIGRAPHX_THROW("Only one output supported"); - if(not sm->bypass()) - MIGRAPHX_THROW("fused_reduce: bypass flag is not set"); - auto names = sm->get_parameter_names(); - check_shapes{inputs, *this}.has(names.size()).same_ndims(); - std::sort(names.begin(), names.end()); - auto shapes = sm->get_parameter_shapes(); - // Check dimension matches for each input - if(not equal(names, inputs, [&](const auto& name, const auto& input) { - return shapes.at(name).lens() == input.lens(); - })) - MIGRAPHX_THROW("Input dimension does not match the submodule."); - - return shape::from_permutation(sm->get_output_shapes().front().type(), - sm->get_output_shapes().front().lens(), - find_permutation(inputs)); - } - - std::string name() const { return "fused_reduce"; } -}; -MIGRAPHX_REGISTER_OP(fused_reduce); - -/* - * Predicate matcher checks that input and output shapes have the same rank. This is assumed - * for broadcast instructions for these fusions. - */ -MIGRAPHX_PRED_MATCHER(input_output_ndim_match, instruction_ref ins) -{ - auto input_shape = ins->inputs().front()->get_shape(); - auto output_shape = ins->get_shape(); - return input_shape.ndim() == output_shape.ndim(); -} - -static auto -insert_module_in_submodule(module_ref sm, - instruction_ref ins, - std::unordered_map* map_ins = nullptr, - module::inserter insert = nullptr) -{ - assert(ins->module_inputs().size() == 1); - return sm->fuse(*ins->module_inputs().front(), ins->inputs(), map_ins, std::move(insert)); -} - -static void create_reduce_modules(module_pass_manager& mpm) -{ - std::size_t n = 0; - for(auto ins : iterator_for(mpm.get_module())) - { - if(not ins->get_operator().attributes().get("reduce", false)) - continue; - if(ins->inputs().size() != 1) - continue; - - auto* rm = - mpm.create_module(mpm.get_module().name() + ":" + ins->name() + std::to_string(n++)); - rm->set_bypass(); - - rm->add_return(rm->fuse({ins})); - auto v = ins->get_operator().to_value(); - mpm.get_module().replace_instruction( - ins, make_op("fused_reduce", {{"axes", v["axes"]}}), ins->inputs(), {rm}); - } -} - -namespace { - -instruction_ref get_broadcast_output(instruction_ref broadcast) -{ - if(broadcast->outputs().size() != 1) - return broadcast; - auto output = broadcast->outputs().front(); - if(output->name() == "contiguous") - return get_broadcast_output(output); - return output; -} - -MIGRAPHX_PRED_MATCHER(used_once_except_broadcast, instruction_ref ins) -{ - if(ins->outputs().size() == 1) - return true; - if(ins->outputs().size() == 2) - { - auto is_broadcast = [](instruction_ref output) { - return contains(output->name(), "broadcast"); - }; - auto broadcast = std::find_if(ins->outputs().begin(), ins->outputs().end(), is_broadcast); - if(broadcast == ins->outputs().end()) - return false; - auto non_broadcast = - std::find_if_not(ins->outputs().begin(), ins->outputs().end(), is_broadcast); - if(non_broadcast == ins->outputs().end()) - return false; - auto output = get_broadcast_output(*broadcast); - return output == *non_broadcast; - } - - return false; -} -} // namespace -template -static auto match_broadcast(Ms... ms) -{ - return match::skip(match::name("contiguous"))( - match::name("multibroadcast")( - match::arg(0)(ms...), match::used_once(), input_output_ndim_match()) - .bind("broadcast")) - .bind("final_broadcast"); -} - -template -static auto any_input(Ms... ms) -{ - return match::any_of[match::inputs()](match::any(ms...).bind("input")); -} - -bool is_valid_broadcast(const instruction_ref b, const std::vector& reduce_axes) -{ - std::vector broadcast_axes; - auto bstrides = b->get_shape().strides(); - - for(size_t i = 0; i < bstrides.size(); ++i) - { - if(bstrides.at(i) == 0) - broadcast_axes.push_back(i); - } - - return broadcast_axes == reduce_axes; -} - -template -static auto match_broadcast_axes(M m) -{ - return match::make_basic_fun_matcher( - [=](match::matcher_context& ctx, instruction_ref ins) -> optional { - optional result = m.match(ctx, ins); - if(contains(ctx.instructions, "broadcast")) - { - instruction_ref reduce; - if(ins->get_operator().name() == "fused_reduce") - { - reduce = ins; - } - else - { - assert(contains(ctx.instructions, "reduce")); - reduce = ctx.instructions["reduce"]; - } - auto axes = reduce->get_operator().to_value().at("axes").to_vector(); - auto broadcast = ctx.instructions["broadcast"]; - if(not is_valid_broadcast(broadcast, axes)) - return nullopt; - } - return result; - }); -} - -static auto match_broadcastable_input(const std::string& op, const std::string& name) -{ - auto match_op = match::name(op)(used_once_except_broadcast()).bind(name); - auto match_op_input = any_input(match_op, match::used_once()); - auto broadcast_match_op_input = any_input(match_broadcast(match_op), match::used_once()); - return match::any_of(match_op_input, match_broadcast_axes(broadcast_match_op_input)); -} - -static void finalize_reduce_module(module_ref m) -{ - eliminate_common_subexpression{}.apply(*m); - dead_code_elimination{}.apply(*m); -} - -namespace { -struct find_pointwise_reduce -{ - auto matcher() const - { - // fused_reduce instruction with pointwise inputs. - return match::name("fused_reduce")(match_broadcastable_input("pointwise", "pointwise")); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto reduce = r.result; - auto input = r.instructions["pointwise"]; - const auto* pm = input->module_inputs().front(); - const auto* old_rm = reduce->module_inputs().front(); - - auto* rm = mpm.create_module(pm->name() + ":" + old_rm->name()); - rm->set_bypass(); - std::unordered_map map_ins; - // Insert pointwise - auto rins = rm->fuse({input}, &map_ins).front(); - map_ins[input] = rins; - - if(contains(r.instructions, "broadcast")) - { - auto broadcast = r.instructions["broadcast"]; - auto fbroadcast = r.instructions["final_broadcast"]; - map_ins[broadcast] = rm->fuse({broadcast}, &map_ins).front(); - if(fbroadcast != broadcast) - map_ins[fbroadcast] = map_ins[broadcast]; - } - - // Insert fused_reduce - rm->add_return(insert_module_in_submodule(rm, reduce, &map_ins)); - finalize_reduce_module(rm); - - auto new_inputs = find_inputs(map_ins, &mpm.get_module(), rm); - mpm.get_module().replace_instruction(reduce, reduce->get_operator(), new_inputs, {rm}); - } -}; - -struct find_reduce_pointwise -{ - - auto matcher() const - { - return match::name("pointwise")(match_broadcastable_input("fused_reduce", "reduce")); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto pw = r.result; - auto reduce = r.instructions["reduce"]; - auto input = r.instructions["input"]; - - const auto* pm = pw->module_inputs().front(); - const auto* old_rm = reduce->module_inputs().front(); - auto* rm = mpm.create_module(old_rm->name() + ":" + pm->name()); - rm->set_bypass(); - std::unordered_map map_ins; - // Copy module instructions - insert_module_in_submodule(rm, reduce, &map_ins); - if(contains(r.instructions, "broadcast")) - { - auto broadcast = r.instructions["broadcast"]; - map_ins[broadcast->inputs().front()] = rm->get_returns().front(); - auto bout = rm->fuse({broadcast}, &map_ins); - map_ins[input] = bout.front(); - } - else - { - map_ins[input] = rm->get_returns().front(); - } - - auto out = rm->fuse({pw}, &map_ins); - rm->replace_return(out); - finalize_reduce_module(rm); - - auto new_inputs = find_inputs(map_ins, &mpm.get_module(), rm); - mpm.get_module().replace_instruction(pw, reduce->get_operator(), new_inputs, {rm}); - } -}; - -struct find_reduce_reduce -{ - auto matcher() const - { - return match::name("fused_reduce")(match_broadcastable_input("fused_reduce", "reduce")); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto reduce1 = r.result; - auto reduce2 = r.instructions["reduce"]; - auto input = r.instructions["input"]; - - if(reduce1->get_operator() != reduce2->get_operator()) - return; - - const auto* rm1 = reduce1->module_inputs().front(); - const auto* rm2 = reduce2->module_inputs().front(); - auto* rm = mpm.create_module(rm1->name() + ":" + rm2->name()); - rm->set_bypass(); - - std::unordered_map map_ins; - // Copy reduce1 instructions - insert_module_in_submodule(rm, reduce2, &map_ins); - if(contains(r.instructions, "broadcast")) - { - auto broadcast = r.instructions["broadcast"]; - map_ins[broadcast->inputs().front()] = rm->get_returns().front(); - auto bout = rm->fuse({broadcast}, &map_ins); - map_ins[input] = bout.front(); - } - else - { - map_ins[input] = rm->get_returns().front(); - } - - auto out = insert_module_in_submodule(rm, reduce1, &map_ins); - rm->replace_return(out); - finalize_reduce_module(rm); - - auto new_inputs = find_inputs(map_ins, &mpm.get_module(), rm); - mpm.get_module().replace_instruction(reduce1, reduce1->get_operator(), new_inputs, {rm}); - } -}; - -struct reduce_reshape : rewrite_reshapes_base -{ - static std::string name() { return "fused_reduce"; } - - template - static auto transform_op(Transform t) - { - return [=](module& m, - instruction_ref ins, - const operation& op, - const std::vector& inputs, - const std::vector& mod_args) { - auto new_op = t(op); - return m.insert_instruction(ins, new_op, inputs, mod_args); - }; - } - - template - static instruction_ref insert(module_pass_manager& mpm, - instruction_ref ins, - const std::vector& inputs, - const AxesMap& am) - { - auto op = any_cast(ins->get_operator()); - std::vector axes; - for(auto axis : op.axes) - { - auto new_axes = am.at(axis); - axes.insert(axes.end(), new_axes.begin(), new_axes.end()); - } - std::sort(axes.begin(), axes.end()); - auto dims = base_dims(inputs); - auto* oldm = ins->module_inputs().front(); - auto* sm = mpm.create_module(oldm->name() + "_reshape"); - sm->set_bypass(); - auto outs = sm->fuse(*oldm, inputs, nullptr, transform_op([&](const operation& sop) { - if(contains(sop.name(), "reduce")) - return make_op(sop.name(), {{"axes", axes}}); - if(sop.name() == "multibroadcast") - return make_op("multibroadcast", {{"out_lens", dims}}); - assert(sop.name() == "pointwise"); - return sop; - })); - sm->add_return(outs); - return mpm.get_module().insert_instruction(ins, fused_reduce{axes}, inputs, {sm}); - } - - static std::vector base_dims(const std::vector& inputs) - { - auto input = std::max_element(inputs.begin(), inputs.end(), by(std::less<>{}, [](auto i) { - return i->get_shape().elements(); - })); - return (*input)->get_shape().lens(); - } - - static std::vector base_dims(instruction_ref ins) - { - return base_dims(ins->inputs()); - } -}; - -} // namespace - -void fuse_reduce::apply(module_pass_manager& mpm) const -{ - if(enabled(MIGRAPHX_DISABLE_REDUCE_FUSION{})) - return; - create_reduce_modules(mpm); - mpm.run_pass(dead_code_elimination{}); - for(int i = 0; i < 4; i++) - { - if(enable_rewrite_reshapes) - mpm.run_pass(rewrite_reshapes{}); - match::find_matches( - mpm, find_reduce_pointwise{}, find_pointwise_reduce{}, find_reduce_reduce{}); - mpm.run_pass(dead_code_elimination{}); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/generate.cpp b/docker/rocm/migraphx/generate.cpp deleted file mode 100644 index d31f1602d..000000000 --- a/docker/rocm/migraphx/generate.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -argument fill_argument(shape s, double value) -{ - argument result; - if(s.type() == shape::tuple_type) - { - std::vector sub_args; - const auto& sub_ss = s.sub_shapes(); - std::transform(sub_ss.begin(), sub_ss.end(), std::back_inserter(sub_args), [&](auto ss) { - return fill_argument(ss, value); - }); - - result = argument(sub_args); - } - else - { - s.visit_type([&](auto as) { - using type = typename decltype(as)::type; - auto v = fill_tensor_data(s, value); - result = {s, v}; - }); - } - return result; -} - -argument generate_argument(shape s, unsigned long seed, random_mode m) -{ - argument result; - if(s.type() == shape::tuple_type) - { - const auto& sub_ss = s.sub_shapes(); - std::vector sub_args; - std::transform(sub_ss.begin(), sub_ss.end(), std::back_inserter(sub_args), [&](auto ss) { - return generate_argument(ss, seed, m); - }); - - result = argument(sub_args); - } - else - { - s.visit_type([&](auto as) { - // we use char type to store bool type internally, so bool_type - // needs special processing to generate data - if(s.type() == shape::bool_type) - { - auto v = generate_tensor_data(s, seed, m); - result = {s, v}; - } - else - { - using type = typename decltype(as)::type; - auto v = generate_tensor_data(s, seed, m); - result = {s, v}; - } - }); - } - - return result; -} - -literal generate_literal(shape s, unsigned long seed) -{ - literal result; - s.visit_type([&](auto as) { - using type = typename decltype(as)::type; - auto v = generate_tensor_data(s, seed); - result = {s, reinterpret_cast(v.get())}; - }); - return result; -} - -// TODO: Move to literal.cpp -literal abs(literal l) -{ - return transform(std::move(l), [](auto x) { return std::fabs(x); }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/include/migraphx/adjust_allocation.hpp b/docker/rocm/migraphx/include/migraphx/adjust_allocation.hpp deleted file mode 100644 index bcd7b0969..000000000 --- a/docker/rocm/migraphx/include/migraphx/adjust_allocation.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ADJUST_ALLOCATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_ADJUST_ALLOCATION_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct MIGRAPHX_EXPORT adjust_allocation -{ - allocation_model model; - std::string name() const { return "adjust_allocation"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/algorithm.hpp b/docker/rocm/migraphx/include/migraphx/algorithm.hpp deleted file mode 100644 index d8f5f0d23..000000000 --- a/docker/rocm/migraphx/include/migraphx/algorithm.hpp +++ /dev/null @@ -1,205 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ALGORITHM_HPP -#define MIGRAPHX_GUARD_RTGLIB_ALGORITHM_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -void transform_if(Iterator start, Iterator last, Output out, Predicate pred, F f) -{ - while(start != last) - { - if(pred(*start)) - { - *out = f(*start); - ++out; - } - ++start; - } -} - -/// Similiar to std::accumulate but a projection can be applied to the elements first -template -T transform_accumulate(Iterator first, Iterator last, T init, BinaryOp binop, UnaryOp unaryop) -{ - return std::inner_product( - first, last, first, init, binop, [&](auto&& x, auto&&) { return unaryop(x); }); -} - -/// Similiar to std::partial_sum but a projection can be applied to the elements first -template -OutputIterator transform_partial_sum( - Iterator first, Iterator last, OutputIterator d_first, BinaryOperation binop, UnaryOp unaryop) -{ - if(first == last) - return d_first; - - auto acc = unaryop(*first); - *d_first = acc; - - while(++first != last) - { - acc = binop(std::move(acc), unaryop(*first)); - *++d_first = acc; - } - - return ++d_first; -} - -template -void group_by(Iterator start, Iterator last, Output out, Predicate pred) -{ - while(start != last) - { - auto it = std::partition(start, last, [&](auto&& x) { return pred(x, *start); }); - out(start, it); - start = it; - } -} - -template -void group_unique(Iterator start, Iterator last, Output out, Predicate pred) -{ - while(start != last) - { - auto it = std::find_if(start, last, [&](auto&& x) { return not pred(*start, x); }); - out(start, it); - start = it; - } -} - -template -void group_find(Iterator start, Iterator last, Predicate pred, Output out) -{ - start = std::find_if(start, last, pred); - while(start != last) - { - auto it = std::find_if_not(start, last, pred); - out(start, it); - start = std::find_if(it, last, pred); - } -} - -/// Similiar to std::remove_if but instead pass adjacent pairs to the predicate -template -Iterator adjacent_remove_if(Iterator first, Iterator last, Predicate p) -{ - first = std::adjacent_find(first, last, p); - if(first == last) - return first; - auto i = first; - while(std::next(++i) != last) - { - if(not p(*i, *std::next(i))) - { - *first = std::move(*i); - ++first; - } - } - *first = std::move(*i); - ++first; - return first; -} - -/// Similiar to std::for_each but instead pass adjacent pairs to the function -template -Iterator adjacent_for_each(Iterator first, Iterator last, F f) -{ - if(first == last) - return last; - - Iterator next = first; - ++next; - - for(; next != last; ++next, ++first) - f(*first, *next); - - return last; -} - -template -std::ptrdiff_t -levenshtein_distance(Iterator1 first1, Iterator1 last1, Iterator2 first2, Iterator2 last2) -{ - if(first1 == last1) - return std::distance(first2, last2); - if(first2 == last2) - return std::distance(first1, last1); - if(*first1 == *first2) - return levenshtein_distance(std::next(first1), last1, std::next(first2), last2); - auto x1 = levenshtein_distance(std::next(first1), last1, std::next(first2), last2); - auto x2 = levenshtein_distance(first1, last1, std::next(first2), last2); - auto x3 = levenshtein_distance(std::next(first1), last1, first2, last2); - return std::ptrdiff_t{1} + std::min({x1, x2, x3}); -} - -inline size_t levenshtein_distance(const std::string& s1, const std::string& s2) -{ - const size_t l1 = s1.length(); - const size_t l2 = s2.length(); - - if(l1 < l2) - levenshtein_distance(s2, s1); - - std::vector d(l2 + 1); - - std::iota(d.begin(), d.end(), 0); - - for(size_t i = 1; i <= l1; i++) - { - size_t prev_cost = d[0]; - d[0] = i; - - for(size_t j = 1; j <= l2; j++) - { - if(s1[i - 1] == s2[j - 1]) - { - d[j] = prev_cost; - } - else - { - size_t cost_insert_or_delete = std::min(d[j - 1], d[j]); - size_t cost_substitute = prev_cost; - prev_cost = d[j]; - d[j] = std::min(cost_substitute, cost_insert_or_delete) + 1; - } - } - } - - return d[l2]; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/allocation_model.hpp b/docker/rocm/migraphx/include/migraphx/allocation_model.hpp deleted file mode 100644 index 9fda8eff9..000000000 --- a/docker/rocm/migraphx/include/migraphx/allocation_model.hpp +++ /dev/null @@ -1,352 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_ALLOCATION_MODEL_HPP -#define MIGRAPHX_GUARD_ALLOCATION_MODEL_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef DOXYGEN - -/// An interface for target-dependent allocation -struct allocation_model -{ - /// A name of the target-dependent allocate operator - std::string name() const; - /// A name of the target-dependent copy operator - std::string copy() const; - /// Create an allocation operator for the given shape - operation allocate(const shape& s) const; - /// Create a preallocated operator for the given shape - operation preallocate(const shape& s, const std::string& id) const; - /// Check if outputs are to be inserted - bool needs_out_params() const; -}; - -#else - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT allocation_model -{ - // - std::string name() const; - // - std::string copy() const; - // - operation allocate(const shape& s) const; - // - operation preallocate(const shape& s, std::string id) const; - // - bool needs_out_params() const; -}; - -#else - -struct allocation_model -{ - private: - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().name(), - std::declval().copy(), - std::declval().allocate(std::declval()), - std::declval().preallocate(std::declval(), - std::declval()), - std::declval().needs_out_params(), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - allocation_model() = default; - - template < - typename PrivateDetailTypeErasedT, - typename = private_te_constraints, - typename = typename std::enable_if< - not std::is_same, allocation_model>{}>::type> - allocation_model(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template < - typename PrivateDetailTypeErasedT, - typename = private_te_constraints, - typename = typename std::enable_if< - not std::is_same, allocation_model>{}>::type> - allocation_model& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - allocation_model rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::string name() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().name(); - } - - std::string copy() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().copy(); - } - - operation allocate(const shape& s) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().allocate(s); - } - - operation preallocate(const shape& s, std::string id) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().preallocate(s, std::move(id)); - } - - bool needs_out_params() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().needs_out_params(); - } - - friend bool is_shared(const allocation_model& private_detail_x, - const allocation_model& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::string name() const = 0; - virtual std::string copy() const = 0; - virtual operation allocate(const shape& s) const = 0; - virtual operation preallocate(const shape& s, std::string id) const = 0; - virtual bool needs_out_params() const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::string name() const override { return private_detail_te_value.name(); } - - std::string copy() const override { return private_detail_te_value.copy(); } - - operation allocate(const shape& s) const override - { - - return private_detail_te_value.allocate(s); - } - - operation preallocate(const shape& s, std::string id) const override - { - - return private_detail_te_value.preallocate(s, std::move(id)); - } - - bool needs_out_params() const override - { - - return private_detail_te_value.needs_out_params(); - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const allocation_model* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(allocation_model* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(allocation_model& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const allocation_model& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/analyze_streams.hpp b/docker/rocm/migraphx/include/migraphx/analyze_streams.hpp deleted file mode 100644 index 6141ca10e..000000000 --- a/docker/rocm/migraphx/include/migraphx/analyze_streams.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ANALYZE_STREAMS_HPP -#define MIGRAPHX_GUARD_RTGLIB_ANALYZE_STREAMS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct stream_race -{ - instruction_ref ins; - // The instruction that should before - instruction_ref before; -}; - -MIGRAPHX_EXPORT std::vector analyze_streams(const module& m, - const stream_model& strmm); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/any_ptr.hpp b/docker/rocm/migraphx/include/migraphx/any_ptr.hpp deleted file mode 100644 index 315edf869..000000000 --- a/docker/rocm/migraphx/include/migraphx/any_ptr.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_ANY_PTR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_ANY_PTR_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct any_ptr -{ - any_ptr() = default; - template - any_ptr(T* p) : ptr(p), ti(typeid(T*)), name(get_name()) - { - } - - any_ptr(void* p, std::string_view pname) : ptr(p), name(pname) {} - - void* get(std::string_view n) const - { - if(name != n) - MIGRAPHX_THROW("any_ptr: type mismatch: " + std::string{name} + - " != " + std::string{n}); - return ptr; - } - - template - T get() const - { - static_assert(std::is_pointer{}, "Must be a pointer"); - assert(ptr != nullptr); - if(ti and std::type_index{typeid(T)} != *ti) - MIGRAPHX_THROW("any_ptr: type mismatch: " + std::string{name} + " != " + get_name()); - else if(name != get_name()) - MIGRAPHX_THROW("any_ptr: type mismatch: " + std::string{name} + " != " + get_name()); - return reinterpret_cast(ptr); - } - void* unsafe_get() const { return ptr; } - - private: - void* ptr = nullptr; - optional ti = nullopt; - std::string_view name = ""; - - template - static const std::string& get_name() - { - return get_type_name>>(); - } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_ANY_PTR_HPP diff --git a/docker/rocm/migraphx/include/migraphx/apply_alpha_beta.hpp b/docker/rocm/migraphx/include/migraphx/apply_alpha_beta.hpp deleted file mode 100644 index 4ed4ebeb6..000000000 --- a/docker/rocm/migraphx/include/migraphx/apply_alpha_beta.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_APPLY_ALPHA_BETA_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_APPLY_ALPHA_BETA_HPP - -#include "migraphx/make_op.hpp" -#include "migraphx/normalize_attributes.hpp" -#include "migraphx/operation.hpp" -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT -instruction_ref insert_apply_alpha_beta(module& m, - instruction_ref pos, - const std::vector& args, - const operation& op, - const literal& alpha, - const literal& beta); - -template -instruction_ref insert_apply_alpha_beta(module& m, - instruction_ref pos, - const std::vector& args, - const operation& op, - T alpha = 1, - T beta = 0) -{ - return insert_apply_alpha_beta(m, pos, args, op, literal{T{alpha}}, literal{T{beta}}); -} - -template -instruction_ref add_apply_alpha_beta(module& m, - const std::vector& args, - const operation& op, - T alpha = 1, - T beta = 0) -{ - return insert_apply_alpha_beta(m, m.end(), args, op, alpha, beta); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_APPLY_ALPHA_BETA_HPP diff --git a/docker/rocm/migraphx/include/migraphx/argument.hpp b/docker/rocm/migraphx/include/migraphx/argument.hpp deleted file mode 100644 index d02a009e3..000000000 --- a/docker/rocm/migraphx/include/migraphx/argument.hpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_ARGUMENT_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_ARGUMENT_HPP - -#include -#include -#include -#include -#include -#include - -// clang-format off -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * @brief Arguments passed to instructions - * - * An `argument` can represent a raw buffer of data that either be referenced from another element - * or it can be owned by the argument. - * - */ -struct MIGRAPHX_EXPORT argument : raw_data -{ - argument() = default; - - explicit argument(const shape& s); - - template ()())>{})> - argument(shape s, F d) - : m_shape(std::move(s)) - - { - assign_buffer([f = std::move(d)]() mutable { return reinterpret_cast(f()); }); - } - template - argument(shape s, T* d) - : m_shape(std::move(s)) - { - assign_buffer([d] { return reinterpret_cast(d); }); - } - - template - argument(shape s, std::shared_ptr d) - : m_shape(std::move(s)) - { - assign_buffer([d] { return reinterpret_cast(d.get()); }); - } - - argument(shape s, std::nullptr_t); - - argument(const std::vector& args); - - /// Provides a raw pointer to the data - char* data() const; - - /// Whether data is available - bool empty() const; - - const shape& get_shape() const; - - argument reshape(const shape& s) const; - - argument copy() const; - - /// Make copy of the argument that is always sharing the data - argument share() const; - - std::vector get_sub_objects() const; - - /// Return the ith element - argument element(std::size_t i) const; - - // Keeps the same data ordering as the given container - template - void fill(Iterator start, Iterator end) - { - assert(std::distance(start, end) <= m_shape.elements()); - this->visit([&](auto output) { - std::copy(start, end, output.begin()); - }); - } - - private: - void assign_buffer(std::function d); - struct data_t - { - std::function get = nullptr; - std::vector sub = {}; - data_t share() const; - static data_t from_args(const std::vector& args); - }; - argument(const shape& s, const data_t& d); - shape m_shape; - data_t m_data{}; -}; - -MIGRAPHX_EXPORT std::vector flatten(const std::vector& args); - -MIGRAPHX_EXPORT std::vector to_shapes(const std::vector& args); -MIGRAPHX_EXPORT void migraphx_to_value(value& v, const argument& a); -MIGRAPHX_EXPORT void migraphx_from_value(const value& v, argument& a); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -// clang-format on - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/array.hpp b/docker/rocm/migraphx/include/migraphx/array.hpp deleted file mode 100644 index e0cf2e3ee..000000000 --- a/docker/rocm/migraphx/include/migraphx/array.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ARRAY_HPP -#define MIGRAPHX_GUARD_RTGLIB_ARRAY_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace detail { - -template -struct array_type -{ - using type = R; -}; -template -struct array_type : std::common_type -{ -}; - -template -using array_type_t = typename array_type::type; - -template -constexpr std::array, N> to_array_impl(T (&a)[N], seq) -{ - return {{a[I]...}}; -} - -} // namespace detail - -template 0))> -constexpr std::array, sizeof...(Ts)> make_array(Ts&&... xs) -{ - return {static_cast>(std::forward(xs))...}; -} - -constexpr std::array make_array() { return {}; } - -template -constexpr auto to_array(T (&a)[N]) -{ - return detail::to_array_impl(a, detail::gens{}); -} - -namespace detail { - -template -constexpr auto rearray_impl(Array a, seq) -{ - return make_array(a[I + Offset]...); -} - -} // namespace detail - -template -constexpr auto pop_front(std::array a) -{ - return detail::rearray_impl(a, detail::gens{}); -} - -template -constexpr auto pop_back(std::array a) -{ - return detail::rearray_impl<1>(a, detail::gens{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/as_number.hpp b/docker/rocm/migraphx/include/migraphx/as_number.hpp deleted file mode 100644 index 987808b60..000000000 --- a/docker/rocm/migraphx/include/migraphx/as_number.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_AS_NUMBER_HPP -#define MIGRAPHX_GUARD_RTGLIB_AS_NUMBER_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -T as_number(T x) -{ - return x; -} -inline int32_t as_number(int8_t x) { return static_cast(x); } -inline uint32_t as_number(uint8_t x) { return static_cast(x); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_RTGLIB_AS_NUMBER_HPP diff --git a/docker/rocm/migraphx/include/migraphx/assert.hpp b/docker/rocm/migraphx/include/migraphx/assert.hpp deleted file mode 100644 index 0f4c2ffa8..000000000 --- a/docker/rocm/migraphx/include/migraphx/assert.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_ASSERT_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_ASSERT_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -auto abort_on_throw(F f) -> decltype(f()) -{ - try - { - return f(); - } - catch(const std::exception& e) - { - std::cerr << e.what() << std::endl; - std::abort(); - } - catch(...) - { - std::cerr << "Unknown exception" << std::endl; - std::abort(); - } -} -#ifdef NDEBUG -#define MIGRAPHX_ASSERT_NO_THROW(...) __VA_ARGS__ -#else -#define MIGRAPHX_ASSERT_NO_THROW(...) \ - migraphx::abort_on_throw([&]() -> decltype(__VA_ARGS__) { return __VA_ARGS__; }) -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_ASSERT_HPP diff --git a/docker/rocm/migraphx/include/migraphx/assignment_options.hpp b/docker/rocm/migraphx/include/migraphx/assignment_options.hpp deleted file mode 100644 index 3eecc0c45..000000000 --- a/docker/rocm/migraphx/include/migraphx/assignment_options.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ASSIGNMENT_OPTIONS_HPP -#define MIGRAPHX_GUARD_RTGLIB_ASSIGNMENT_OPTIONS_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct assignment_options -{ - support_metric metric = support_metric::latency; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_RTGLIB_ASSIGNMENT_OPTIONS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/auto_any_cast.hpp b/docker/rocm/migraphx/include/migraphx/auto_any_cast.hpp deleted file mode 100644 index ec0c51469..000000000 --- a/docker/rocm/migraphx/include/migraphx/auto_any_cast.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_AUTO_ANY_CAST_HPP -#define MIGRAPHX_GUARD_RTGLIB_AUTO_ANY_CAST_HPP -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// Forward declare any_cast -template -const T& any_cast(const T&); - -namespace detail { - -template -void any_cast() -{ -} - -template -struct auto_any_caster -{ - T& x; // NOLINT - - template - operator U&() - { - return any_cast(x); - } - - operator T&() { return x; } -}; - -} // namespace detail - -template -detail::auto_any_caster auto_any_cast(T& x) -{ - return {x}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/auto_contiguous.hpp b/docker/rocm/migraphx/include/migraphx/auto_contiguous.hpp deleted file mode 100644 index 25fa67b16..000000000 --- a/docker/rocm/migraphx/include/migraphx/auto_contiguous.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_AUTO_CONTIGOUS_HPP -#define MIGRAPHX_GUARD_RTGLIB_AUTO_CONTIGOUS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct MIGRAPHX_EXPORT auto_contiguous -{ - std::string name() const { return "auto_contiguous"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/auto_register.hpp b/docker/rocm/migraphx/include/migraphx/auto_register.hpp deleted file mode 100644 index d40791cf4..000000000 --- a/docker/rocm/migraphx/include/migraphx/auto_register.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_AUTO_REGISTER_HPP -#define MIGRAPHX_GUARD_RTGLIB_AUTO_REGISTER_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -int auto_register_action() -{ - Action::template apply(); - return 0; -} - -template -struct auto_register -{ - const static int static_register; - // This typedef ensures that the static member will be instantiated if - // the class itself is instantiated - using static_register_type = - std::integral_constant; -}; - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wglobal-constructors" -#endif - -template -const int auto_register::static_register = auto_register_action(); // NOLINT - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -#define MIGRAPHX_AUTO_REGISTER_NAME_DETAIL(x) migraphx_auto_register_##x -#define MIGRAPHX_AUTO_REGISTER_NAME(x) MIGRAPHX_AUTO_REGISTER_NAME_DETAIL(x) -// NOLINTNEXTLINE -#define MIGRAPHX_AUTO_REGISTER(...) \ - [[maybe_unused]] void MIGRAPHX_AUTO_REGISTER_NAME(__LINE__)( \ - migraphx::auto_register<__VA_ARGS__> x = migraphx::auto_register<__VA_ARGS__>{}); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/autocast_fp8.hpp b/docker/rocm/migraphx/include/migraphx/autocast_fp8.hpp deleted file mode 100644 index cf0d2a49b..000000000 --- a/docker/rocm/migraphx/include/migraphx/autocast_fp8.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_AUTOCAST_FP8_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_AUTOCAST_FP8_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct program; -struct module; - -/** -This pass will convert model with fp8 input parameter to model with fp32 -input parameter and internally add casts to fp8 for those converted params.*/ -struct MIGRAPHX_EXPORT autocast_fp8_pass -{ - shape::type_t target_type = migraphx::shape::float_type; - std::string name() const { return "autocast_fp8_pass"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/base64.hpp b/docker/rocm/migraphx/include/migraphx/base64.hpp deleted file mode 100644 index 2af7a60b2..000000000 --- a/docker/rocm/migraphx/include/migraphx/base64.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_BASE64_HPP -#define MIGRAPHX_GUARD_RTGLIB_BASE64_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// encode string to base64 -std::string MIGRAPHX_EXPORT base64_encode(const std::string& str); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/bf16.hpp b/docker/rocm/migraphx/include/migraphx/bf16.hpp deleted file mode 100644 index 26ecdd7c9..000000000 --- a/docker/rocm/migraphx/include/migraphx/bf16.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_BF16_HPP -#define MIGRAPHX_GUARD_RTGLIB_BF16_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using bf16 = migraphx::generic_float<7, 8>; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/bit_cast.hpp b/docker/rocm/migraphx/include/migraphx/bit_cast.hpp deleted file mode 100644 index fc4aab2e3..000000000 --- a/docker/rocm/migraphx/include/migraphx/bit_cast.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/* ************************************************************************ - * Copyright (C) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- - * ies of the Software, and to permit persons to whom the Software is furnished - * to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- - * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- - * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ************************************************************************ */ -#ifndef MIGRAPHX_GUARD_RTGLIB_BITCAST_HPP -#define MIGRAPHX_GUARD_RTGLIB_BITCAST_HPP -#include -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstrict-aliasing" -#pragma GCC diagnostic ignored "-Wduplicated-branches" -#endif - -#include -#include - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define MIGRAPHX_CONST_FOLD(x) (__builtin_constant_p(x) ? (x) : (x)) - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -template {} and - std::is_trivially_copyable{})> -inline constexpr To bit_cast(From fr) noexcept -{ - static_assert(sizeof(To) == sizeof(From)); -#if defined(__GNUC__) and !defined(__clang__) - return MIGRAPHX_CONST_FOLD(*reinterpret_cast(&fr)); -#else - return __builtin_bit_cast(To, fr); -#endif -} -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif -#endif // MIGRAPHX_GUARD_RTGLIB_BITCAST_HPP diff --git a/docker/rocm/migraphx/include/migraphx/bit_signal.hpp b/docker/rocm/migraphx/include/migraphx/bit_signal.hpp deleted file mode 100644 index 97a166cb2..000000000 --- a/docker/rocm/migraphx/include/migraphx/bit_signal.hpp +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_BIT_SIGNAL_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_BIT_SIGNAL_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// Observer pattern for keeping track of if something has changed or been -/// updated. Can have up to `N` different subscribers. Use by creating a -/// `bit_signal` and adding subscribers with `bit_signal.subscribe()`. Use -/// `bit_signal.notify()` to set that subscribers should be notified. Get the -/// status of the subscription by checking the `slot` returned by -/// `bit_signal.subscribe()`. -template -struct bit_signal -{ - std::bitset slots; - std::bitset allocated; - - struct slot - { - bit_signal* handler = nullptr; - std::size_t i = N; - - slot() = default; - - slot(bit_signal* h, std::size_t x) : handler(h), i(x) {} - - slot(slot&& rhs) noexcept : handler(rhs.handler), i(rhs.i) - { - rhs.handler = nullptr; - rhs.i = N; - } - - slot(const slot& rhs) : handler(rhs.handler), i(rhs.handler->allocate()) {} - - slot& operator=(slot rhs) - { - std::swap(handler, rhs.handler); - std::swap(i, rhs.i); - return *this; - } - - ~slot() noexcept - { - if(valid()) - handler->deallocate(i); - } - - bool valid() const { return i < N and handler != nullptr; } - - bool triggered() const - { - assert(valid()); - return handler->triggered(i); - } - - operator bool() const { return triggered(); } - }; - - slot subscribe() { return {this, allocate()}; } - - std::size_t allocate() - { - auto i = *find_if(range(N), [&](auto x) { return not allocated[x]; }); - if(i == N) - MIGRAPHX_THROW("Too many signals allocated"); - slots[i] = false; - allocated[i] = true; - return i; - } - - void deallocate(std::size_t i) { allocated[i] = false; } - - void notify() { slots.set(); } - - bool triggered(std::size_t i) const { return slots[i]; } - - void clear() - { - slots.reset(); - allocated.reset(); - } - - std::size_t nslots() const { return allocated.count(); } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_BIT_SIGNAL_HPP diff --git a/docker/rocm/migraphx/include/migraphx/builtin.hpp b/docker/rocm/migraphx/include/migraphx/builtin.hpp deleted file mode 100644 index fb5d16c32..000000000 --- a/docker/rocm/migraphx/include/migraphx/builtin.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_BUILTIN_HPP -#define MIGRAPHX_GUARD_BUILTIN_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace builtin { - -struct literal -{ - std::string name() const { return "@literal"; } - shape compute_shape(const std::vector&) const { MIGRAPHX_THROW("builtin"); } - argument compute(context&, const shape&, const std::vector&) const - { - MIGRAPHX_THROW("builtin"); - } -}; - -struct outline -{ - shape s; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape")); - } - - std::string name() const { return "@outline"; } - shape compute_shape(const std::vector&) const { return s; } - argument compute(context&, const shape&, const std::vector&) const - { - MIGRAPHX_THROW("builtin"); - } -}; - -struct param -{ - std::string parameter; - uint32_t order = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.parameter, "parameter")); - } - - std::string name() const { return "@param"; } - shape compute_shape(const std::vector&) const { MIGRAPHX_THROW("builtin"); } - argument compute(context&, const shape&, const std::vector&) const - { - MIGRAPHX_THROW("builtin"); - } - friend std::ostream& operator<<(std::ostream& os, const param& op) - { - os << op.name() << ":" << op.parameter; - return os; - } -}; - -struct returns -{ - std::string name() const { return "@return"; } - - shape compute_shape(const std::vector& arg) const - { - if(arg.empty()) - return {}; - else if(arg.size() == 1) - return arg[0]; - else - return shape(arg); - } - - argument compute(context&, const shape&, const std::vector&) const - { - MIGRAPHX_THROW("builtin"); - } -}; - -} // namespace builtin -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/check_context.hpp b/docker/rocm/migraphx/include/migraphx/check_context.hpp deleted file mode 100644 index 9edb6fa7b..000000000 --- a/docker/rocm/migraphx/include/migraphx/check_context.hpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CHECK_CONTEXT_HPP -#define MIGRAPHX_GUARD_RTGLIB_CHECK_CONTEXT_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct check_context -{ - struct op : auto_register_op - { - static std::string compute_op_name() - { - const auto& op_type_name = get_type_name(); - const auto& split_name = split_string(op_type_name, ':'); - std::vector name_without_version = {"check_context"}; - // op_type_name would contain internal namespace name with version_x_y_z - // remove version and construct op_name such as check_context::migraphx::gpu::context - std::copy_if( - split_name.begin(), - split_name.end(), - std::back_inserter(name_without_version), - [&](const auto& i) { return not i.empty() and not contains(i, "version"); }); - return join_strings(name_without_version, "::"); - } - - std::string name() const - { - static auto op_name = compute_op_name(); - return op_name; - } - - shape compute_shape(const std::vector&) const { return {}; } - argument compute(context& ctx, const shape&, const std::vector&) const - { - this->check(ctx); - return {}; - } - void finalize(context& ctx, const shape&, const std::vector&) const - { - this->check(ctx); - } - void check(context& ctx) const - { - T* x = any_cast(&ctx); - if(x == nullptr) - MIGRAPHX_THROW(std::string("Unexpected context type: ") + ctx.type_id().name()); - } - }; - - std::string name() const { return "check_context"; } - void apply(module& m) const { m.insert_instruction(m.begin(), op{}); } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/check_shapes.hpp b/docker/rocm/migraphx/include/migraphx/check_shapes.hpp deleted file mode 100644 index cbffb7580..000000000 --- a/docker/rocm/migraphx/include/migraphx/check_shapes.hpp +++ /dev/null @@ -1,431 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CHECK_SHAPES_HPP -#define MIGRAPHX_GUARD_RTGLIB_CHECK_SHAPES_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// Check that deduced type is incrementable, dereferencable, and comparable -template -struct is_iterator -{ -}; - -template -struct is_iterator()), - decltype(*std::declval()), - decltype(std::declval() == std::declval())>> : std::true_type -{ -}; - -template -struct check_shapes -{ - static_assert(is_iterator{}, "CHECK_SHAPES: Deduced type must be an iterator"); - Iterator begin; - Iterator end; - std::string name; - bool dynamic_allowed; - - check_shapes(Iterator b, Iterator e, const std::string& n, const bool d = false) - : begin(b), end(e), name(n), dynamic_allowed(d) - { - check_dynamic(); - } - - template - check_shapes(Iterator b, Iterator e, const Op& op, const bool d = false) - : begin(b), end(e), name(op.name()), dynamic_allowed(d) - { - check_dynamic(); - } - - template {})> - check_shapes(const std::vector& s, const Op& op, const bool d = false) - : begin(s.begin()), end(s.end()), name(op.name()), dynamic_allowed(d) - { - check_dynamic(); - } - - check_shapes(const std::vector& s, const std::string& n, const bool d = false) - : begin(s.begin()), end(s.end()), name(n), dynamic_allowed(d) - { - check_dynamic(); - } - - void check_dynamic() const - { - if(not dynamic_allowed and this->any_of([&](const shape& s) { return s.dynamic(); })) - { - MIGRAPHX_THROW(prefix() + "Dynamic shapes not supported"); - } - } - - std::string prefix() const - { - if(name.empty()) - return ""; - else - return name + ": "; - } - - std::size_t size() const - { - if(begin == end) - return 0; - return end - begin; - } - - /*! - * Require the number of shape objects to equal to one of the - * given sizes. - * \param ns template parameter pack of sizes to check against - */ - template - const check_shapes& has(Ts... ns) const - { - if(migraphx::none_of({ns...}, [&](auto i) { return this->size() == i; })) - MIGRAPHX_THROW(prefix() + "Wrong number of arguments: expected " + - to_string_range({ns...}) + " but given " + std::to_string(size())); - return *this; - } - - /*! - * Require the number of shape objects to equal at least a given amount. Use this - * method for ops that can take any number (variadic) of inputs. - * \param n min. number of shapes - */ - const check_shapes& has_at_least(std::size_t n) const - { - if(this->size() < n) - MIGRAPHX_THROW(prefix() + "Wrong number of arguments: expected at least " + - to_string(n) + " but given " + std::to_string(size())); - return *this; - } - - /*! - * Require all shapes to have the same number of elements. - * \param n number of - */ - const check_shapes& nelements(std::size_t n) const - { - if(not this->all_of([&](const shape& s) { return s.elements() == n; })) - MIGRAPHX_THROW(prefix() + "Shapes must have only " + std::to_string(n) + " elements"); - return *this; - } - - /*! - * Check that the first shape has exactly n dimensions. - * Do nothing if the container is empty. - * \param n number of dimensions - */ - const check_shapes& only_dims(std::size_t n) const - { - if(begin != end) - { - if(begin->ndim() != n) - MIGRAPHX_THROW(prefix() + "Only " + std::to_string(n) + "d supported"); - } - return *this; - } - - /*! - * Check that the first shape has a maximum of n dimensions. - * Do nothing if the container is empty. - * \param n number of dimensions - */ - const check_shapes& max_ndims(std::size_t n) const - { - if(begin != end) - { - if(begin->ndim() > n) - MIGRAPHX_THROW(prefix() + "Shape must have at most " + std::to_string(n) + - " dimensions"); - } - return *this; - } - - /*! - * Check that the first shape has a minimum of n dimensions. - * Do nothing if the container is empty. - * \param n number of dimensions - */ - const check_shapes& min_ndims(std::size_t n) const - { - if(begin != end) - { - if(begin->ndim() < n) - MIGRAPHX_THROW(prefix() + "Shape must have at least " + std::to_string(n) + - " dimensions"); - } - return *this; - } - - /*! - * Check all shapes have the same shape. - */ - const check_shapes& same_shape() const - { - if(not this->same([](const shape& s) { return s; })) - MIGRAPHX_THROW(prefix() + "Shapes do not match"); - return *this; - } - - /*! - * Check all shapes have the same type. - */ - const check_shapes& same_type() const - { - if(not this->same([](const shape& s) { return s.type(); })) - MIGRAPHX_THROW(prefix() + "Types do not match"); - return *this; - } - - /*! - * Check all shapes have the same lens. - */ - const check_shapes& same_dims() const - { - if(not this->same([](const shape& s) { return s.max_lens(); })) - MIGRAPHX_THROW(prefix() + "Dimensions do not match"); - if(this->any_of([&](const shape& s) { return s.dynamic(); })) - if(not this->same([](const shape& s) { return s.min_lens(); })) - MIGRAPHX_THROW(prefix() + "Min dynamic dimensions do not match"); - return *this; - } - - /*! - * Check all shapes have the same number of dimensions. - */ - const check_shapes& same_ndims() const - { - if(not this->same([](const shape& s) { return s.ndim(); })) - MIGRAPHX_THROW(prefix() + "Number of dimensions do not match"); - return *this; - } - - /*! - * Check all shapes have the same layout. - */ - const check_shapes& same_layout() const - { - if(not this->same([](const shape& s) { return find_permutation(s); })) - MIGRAPHX_THROW(prefix() + "Layouts do not match"); - return *this; - } - - /*! - * Check all shapes are standard. - */ - const check_shapes& standard() const - { - if(not this->all_of([](const shape& s) { return s.standard(); })) - MIGRAPHX_THROW(prefix() + "Shapes are not in standard layout"); - return *this; - } - - /*! - * Check all shapes are scalar. - */ - const check_shapes& scalar() const - { - if(not this->all_of([](const shape& s) { return s.scalar(); })) - MIGRAPHX_THROW(prefix() + "Shapes are not a scalar"); - return *this; - } - - /*! - * Check all shapes are standard or scalar. - */ - const check_shapes& standard_or_scalar() const - { - if(not this->all_of([](const shape& s) { return s.standard() or s.scalar(); })) - MIGRAPHX_THROW(prefix() + "Shapes are not a scalar or in standard layout"); - return *this; - } - - /*! - * Check all shapes are packed. - */ - const check_shapes& packed() const - { - if(not this->all_of([](const shape& s) { return s.packed(); })) - MIGRAPHX_THROW(prefix() + "Shapes are not packed"); - return *this; - } - - /*! - * Check all shapes are packed with certain layouts - */ - const check_shapes& - packed_layouts(const std::initializer_list>& layouts) const - { - if(not this->all_of([&](const shape& s) { - return s.packed() and contains(layouts, find_permutation(s)); - })) - MIGRAPHX_THROW(prefix() + "Shapes are not packed with correct layout"); - return *this; - } - - /*! - * Check all shapes are packed or broadcasted. - */ - const check_shapes& packed_or_broadcasted() const - { - if(not this->all_of([](const shape& s) { return s.packed() or s.broadcasted(); })) - MIGRAPHX_THROW(prefix() + "Shapes are not packed nor broadcasted"); - return *this; - } - - /*! - * Check all shapes are tuples. - */ - const check_shapes& tuple_type() const - { - if(not this->all_of([](const shape& s) { return s.type() == shape::tuple_type; })) - MIGRAPHX_THROW(prefix() + "Shapes are not tuple!"); - return *this; - } - - /*! - * Check all shapes are not transposed. - */ - const check_shapes& not_transposed() const - { - if(not this->all_of([](const shape& s) { return not s.transposed(); })) - MIGRAPHX_THROW(prefix() + "Shapes are transposed"); - return *this; - } - - /*! - * Check all shapes are not broadcasted. - */ - const check_shapes& not_broadcasted() const - { - if(not this->all_of([](const shape& s) { return s.standard() or not s.broadcasted(); })) - MIGRAPHX_THROW(prefix() + "Shapes are broadcasted"); - return *this; - } - - /*! - * Check all shapes have the same n elements. - * \param n number of elements - */ - const check_shapes& elements(std::size_t n) const - { - if(not this->all_of([&](const shape& s) { return s.elements() == n; })) - MIGRAPHX_THROW(prefix() + "Wrong number of elements"); - return *this; - } - - /*! - * Check the batches of all the shapes do not have transposed strides. - */ - const check_shapes& batch_not_transposed() const - { - if(not this->all_of( - [&](const shape& s) { return batch_not_transposed_strides(s.strides()); })) - MIGRAPHX_THROW(prefix() + "Batch size is transposed"); - return *this; - } - - template - bool same(F f) const - { - if(begin == end) - return true; - auto&& key = f(*begin); - return this->all_of([&](const shape& s) { return f(s) == key; }); - } - - template - bool all_of(Predicate p) const - { - if(begin == end) - return true; - return std::all_of(begin, end, p); - } - - template - bool any_of(Predicate p) const - { - if(begin == end) - return false; - return std::any_of(begin, end, p); - } - - Iterator get(long i) const - { - if(i >= size()) - MIGRAPHX_THROW(prefix() + "Accessing shape out of bounds"); - if(i < 0) - return end - i; - return begin + i; - } - - check_shapes slice(long start) const { return {get(start), end, name}; } - - check_shapes slice(long start, long last) const { return {get(start), get(last), name}; } - - private: - static bool batch_not_transposed_strides(const std::vector& strides) - { - if(strides.size() <= 2) - return true; - auto dim_0 = strides.size() - 2; - auto matrix_size = std::max(strides[dim_0], strides[dim_0 + 1]); - std::vector batch(strides.begin(), strides.begin() + dim_0); - if(std::all_of(batch.begin(), batch.end(), [&](auto i) { return (i < matrix_size); })) - { - return false; - } - - if(std::adjacent_find(batch.begin(), batch.end(), [&](auto i, auto j) { - return (i < j or i < matrix_size or j < matrix_size); - }) != batch.end()) - { - return false; - } - return true; - } -}; - -// Deduction guide for std::vector constructor -template -check_shapes(const std::vector&, const Op&, bool d = false) - -> check_shapes::const_iterator>; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/clamp.hpp b/docker/rocm/migraphx/include/migraphx/clamp.hpp deleted file mode 100644 index 18475419e..000000000 --- a/docker/rocm/migraphx/include/migraphx/clamp.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CLAMP_HPP -#define MIGRAPHX_GUARD_RTGLIB_CLAMP_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -U pad_clamp(T x) -{ - if(float_equal(x, std::numeric_limits::lowest())) - return std::numeric_limits::lowest(); - if(float_equal(x, std::numeric_limits::max())) - return std::numeric_limits::max(); - return (x < std::numeric_limits::lowest()) - ? std::numeric_limits::lowest() - : (std::numeric_limits::max() < x) ? std::numeric_limits::max() : U(x); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/cloneable.hpp b/docker/rocm/migraphx/include/migraphx/cloneable.hpp deleted file mode 100644 index 820bc4221..000000000 --- a/docker/rocm/migraphx/include/migraphx/cloneable.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CLONEABLE_HPP -#define MIGRAPHX_GUARD_RTGLIB_CLONEABLE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct cloneable -{ - friend Base; - - virtual std::shared_ptr clone() = 0; - - template - struct derive : Base - { - friend Derived; - - std::shared_ptr clone() override - { - return std::make_shared(static_cast(*this)); - } - template - derive(Args&&... args) : Base(std::forward(args)...) - { - } - }; - - struct share : Base, std::enable_shared_from_this - { - std::shared_ptr clone() override { return this->shared_from_this(); } - template - share(Args&&... args) : Base(std::forward(args)...) - { - } - }; - cloneable() = default; - cloneable(const cloneable&) = default; - cloneable& operator=(const cloneable&) = default; - virtual ~cloneable() {} -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/common.hpp b/docker/rocm/migraphx/include/migraphx/common.hpp deleted file mode 100644 index 63a2dc435..000000000 --- a/docker/rocm/migraphx/include/migraphx/common.hpp +++ /dev/null @@ -1,158 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_COMMON_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_COMMON_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -struct operation; - -struct common_options -{ - bool common_type = true; - bool common_lens = true; -}; - -/** - * Broadcasting works by comparing the shapes element-wise starting with - * the trailing (right-most) dimensions and working leftwards. This is equivalent - * to what is done in NumPy. - * example 1: - * s0 = (3,2,4,5) and s1 = (2,1,1) - * In this case we need to broadcast (:,1,1) portion of - * s1 plus broadcast the 1st dimension of s0 - * giving output_lens = (3,2,4,5) - * - * example 2: - * s0 = (3,2,1,5) and s1 = (2,7,5) - * In this case we need to broadcast the (:,:,1:,:) axis - * of s0 plus the 1st dimension of s1 giving - * output_lens = (3,2,7,5) - * - * example 3: - * s0 = (4, 1, 1) and s1 = (3, 4) - * output_lens = (4, 3, 4) - */ -MIGRAPHX_EXPORT -std::vector compute_broadcasted_lens(std::vector s0, - std::vector s1); - -/** - * Broadcasting for two vectors of dynamic_dimensions. - * Compares `dynamic_dimension` objects from the trailing (right-most) dimension and working - * leftwards. - * - * Rules for broadcasting dynamic_dimension: - * If the same `dynamic_dimension`, return either. - * If one of the `dynamic_dimension`s is 1, return the other one. - * If the `dynamic_dimension`s have an intersection, return the intersection. - * Explanation: - * For the shape to be broadcastable at runtime (when the dimensions are constant) the dimensions - * must be the same. The only way for the dimensions to be the same is if the output dimension is - * the intersection of the ranges. - * In practice, we will mostly see this case for handling unknown dynamic_dimensions like {0, - * max_int}. Else, throw an error. - * - * There is a contrived edge case for ranges that include 1 but are not a fixed {1, 1}. - * That case is not supported. - */ -MIGRAPHX_EXPORT -std::vector -compute_broadcasted_dyn_dims(std::vector dds0, - std::vector dds1); - -MIGRAPHX_EXPORT -std::vector compute_broadcasted_dyn_dims(shape s0, shape s1); - -MIGRAPHX_EXPORT -shape common_shape(const std::vector& shapes); - -/** - * @brief Compute the common (broadcasted) dimensions of a list of fixed shapes - */ -MIGRAPHX_EXPORT -std::vector compute_common_lens(const std::vector& shapes); - -/** - * @ brief Compute the common (broadcasted) dynamic dimensions of a list of dynamic shapes - */ -MIGRAPHX_EXPORT -std::vector compute_common_dyn_dims(const std::vector& shapes); - -/** - * @brief Creates and adds instructions to convert input arguments to common shapes and types - * by adding multi-broadcast and type convert operations. This is a utility function for creating - * operations where the shape and type of inputs need to match. It supports both dynamic and - * static-shaped arguments. - * - * @param m containing module for instruction - * @param ins insertion location in instruction list - * @param inputs instructions to use as argument list; also, the shapes - * attached to each instruction_ref are considered for broadcasting - * @return std::vector a modified argument list - */ -MIGRAPHX_EXPORT std::vector insert_common_args(module& m, - instruction_ref ins, - std::vector inputs, - common_options options = {}); - -MIGRAPHX_EXPORT -std::vector -add_common_args(module& m, std::vector inputs, common_options options = {}); - -MIGRAPHX_EXPORT -instruction_ref insert_common_op(module& m, - instruction_ref ins, - const operation& op, - std::vector inputs, - common_options options = {}); - -/** - * @brief Wrapper for insert_common_args() which inserts operation at the end of the module. - */ -MIGRAPHX_EXPORT -instruction_ref add_common_op(module& m, - const operation& op, - std::vector inputs, - common_options options = {}); - -/** - * Calculates the broadcasted shape with the given input_shape and broadcasted dimensions. - * - * @param input_shape static shape to broadcast - * @param bcast_lens dimensions to broadcast to - * @return broadcasted shape with calculated strides - */ -MIGRAPHX_EXPORT -shape make_bcast_shape(const shape& input_shape, const std::vector& bcast_lens); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_COMMON_HPP diff --git a/docker/rocm/migraphx/include/migraphx/common_dims.hpp b/docker/rocm/migraphx/include/migraphx/common_dims.hpp deleted file mode 100644 index b8dd30075..000000000 --- a/docker/rocm/migraphx/include/migraphx/common_dims.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_COMMON_DIMS_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_COMMON_DIMS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// This will compute a higher dimensional space that will preserve the axes -/// for both sets of dimensions. Two axes_maps are provided for each of the -/// dims that will map the axis to the axes that are used by the result of -/// common_dims. -struct MIGRAPHX_EXPORT common_dims -{ - static common_dims compute(const std::vector& dims1, - const std::vector& dims2); - - /// Map the dimensions into the common higher dimensional space. The - /// dimension doesnt need to have the same number of elements as the - /// common dimension. - std::vector get_dimensions_for(const std::vector& idims) const; - /// Get the corresponding axes map based on the rank of tensor - const std::vector>* get_axes_map(std::size_t n) const; - std::vector dims; - std::vector> axes_map1; - std::vector> axes_map2; -}; - -template -auto elements(const Range& r) -{ - return std::accumulate(r.begin(), r.end(), std::size_t{1}, std::multiplies<>{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_COMMON_DIMS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/compile_options.hpp b/docker/rocm/migraphx/include/migraphx/compile_options.hpp deleted file mode 100644 index 64eb3018c..000000000 --- a/docker/rocm/migraphx/include/migraphx/compile_options.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_COMPILE_OPTIONS_HPP -#define MIGRAPHX_GUARD_RTGLIB_COMPILE_OPTIONS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct compile_options -{ - /** - * Have MIGX allocate memory for parameters and add instructions - * to copy parameters and output to/from an offload device like a GPU. - */ - bool offload_copy = false; - - bool fast_math = true; - bool exhaustive_tune = false; - - tracer trace{}; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/compile_src.hpp b/docker/rocm/migraphx/include/migraphx/compile_src.hpp deleted file mode 100644 index 7efd18785..000000000 --- a/docker/rocm/migraphx/include/migraphx/compile_src.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_COMPILE_SRC_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_COMPILE_SRC_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct src_file -{ - fs::path path; - std::string_view content; - - src_file() = default; - src_file(fs::path file_path, std::string_view file_content) - : path{std::move(file_path)}, content{file_content} - { - } - - explicit src_file(const std::pair& pair) - : path{pair.first}, content{pair.second} - { - } -}; - -struct MIGRAPHX_EXPORT src_compiler -{ -#ifdef _WIN32 - fs::path compiler = MIGRAPHX_CXX_COMPILER; -#else - fs::path compiler = "c++"; -#endif - std::vector flags = {}; - fs::path output = {}; - fs::path launcher = {}; - std::string out_ext = ".o"; - std::function process = nullptr; - std::vector compile(const std::vector& srcs) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_COMPILE_SRC_HPP diff --git a/docker/rocm/migraphx/include/migraphx/concat_opt.hpp b/docker/rocm/migraphx/include/migraphx/concat_opt.hpp deleted file mode 100644 index b1295e6d3..000000000 --- a/docker/rocm/migraphx/include/migraphx/concat_opt.hpp +++ /dev/null @@ -1,303 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_CONCAT_OPT_HPP -#define MIGRAPHX_GUARD_CONCAT_OPT_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef DOXYGEN - -/// An interface for target-dependent optimization for the concat instruction -struct concat_optimization -{ - /// A name of the target-dependent allocate operator - std::string allocate() const; - /// Return the target-independent concat operator - optional get_concat(const operation& op) const; -}; - -#else - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT concat_optimization -{ - // - std::string allocate() const; - // - optional get_concat(const operation& op) const; -}; - -#else - -struct concat_optimization -{ - private: - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().allocate(), - std::declval().get_concat( - std::declval()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - concat_optimization() = default; - - template , - typename = typename std::enable_if< - not std::is_same, - concat_optimization>{}>::type> - concat_optimization(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template , - typename = typename std::enable_if< - not std::is_same, - concat_optimization>{}>::type> - concat_optimization& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - concat_optimization rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::string allocate() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().allocate(); - } - - optional get_concat(const operation& op) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_concat(op); - } - - friend bool is_shared(const concat_optimization& private_detail_x, - const concat_optimization& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::string allocate() const = 0; - virtual optional get_concat(const operation& op) const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::string allocate() const override { return private_detail_te_value.allocate(); } - - optional get_concat(const operation& op) const override - { - - return private_detail_te_value.get_concat(op); - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const concat_optimization* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(concat_optimization* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(concat_optimization& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const concat_optimization& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/config.hpp b/docker/rocm/migraphx/include/migraphx/config.hpp deleted file mode 100644 index ffb45a16a..000000000 --- a/docker/rocm/migraphx/include/migraphx/config.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_CONFIG_HPP -#define MIGRAPHX_GUARD_CONFIG_HPP - -#include -#include - -#if !defined(MIGRAPHX_USE_CLANG_TIDY) && !defined(DOXYGEN) - -#ifdef BUILD_DEV -#define MIGRAPHX_INLINE_NS version_1 -#else -#include - -#define MIGRAPHX_VERSION_PRIMITIVE_CONCAT(x, y) x##_##y -#define MIGRAPHX_VERSION_CONCAT(x, y) MIGRAPHX_VERSION_PRIMITIVE_CONCAT(x, y) - -#define MIGRAPHX_VERSION \ - MIGRAPHX_VERSION_CONCAT( \ - MIGRAPHX_VERSION_CONCAT(MIGRAPHX_VERSION_MAJOR, MIGRAPHX_VERSION_MINOR), \ - MIGRAPHX_VERSION_PATCH) - -#define MIGRAPHX_INLINE_NS MIGRAPHX_VERSION_CONCAT(version, MIGRAPHX_VERSION) -#endif // build_dev -#endif // clang_tidy - -#ifdef DOXYGEN -#define MIGRAPHX_INLINE_NS internal -#endif // doxygen - -#ifdef MIGRAPHX_USE_CLANG_TIDY -#define MIGRAPHX_TIDY_CONST const -#else -#define MIGRAPHX_TIDY_CONST -#endif // tidy_const -#endif // clang_tidy diff --git a/docker/rocm/migraphx/include/migraphx/context.hpp b/docker/rocm/migraphx/include/migraphx/context.hpp deleted file mode 100644 index 88d42f666..000000000 --- a/docker/rocm/migraphx/include/migraphx/context.hpp +++ /dev/null @@ -1,466 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_CONTEXT_HPP -#define MIGRAPHX_GUARD_CONTEXT_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef DOXYGEN - -/// A context is used to store internal data for a `target`. A context is -/// constructed by a target during compilation and passed to the operations -/// during `eval`. -struct context -{ - /// Wait for any tasks in the context to complete - void finish() const; -}; - -#else - -template -value to_value_context(const T&) -{ - return value{}; -} - -template -void from_value_context(T&, const value&) -{ -} - -template -any_ptr get_queue_context(T&) -{ - return {}; -} - -template -void wait_for_context(T&, any_ptr) -{ -} - -template -void finish_on_context(T&, any_ptr) -{ -} - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT context -{ - // (optional) - value to_value() const; - // (optional) - void from_value(const value& v); - // (optional) - any_ptr get_queue(); - // (optional) - void wait_for(any_ptr queue); - // (optional) - void finish_on(any_ptr queue); - // - void finish() const; -}; - -#else - -struct context -{ - private: - template - static auto private_detail_te_default_to_value(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.to_value()) - { - return private_detail_te_self.to_value(); - } - - template - static value private_detail_te_default_to_value(float, T&& private_detail_te_self) - { - return to_value_context(private_detail_te_self); - } - - template - static auto - private_detail_te_default_from_value(char, T&& private_detail_te_self, const value& v) - -> decltype(private_detail_te_self.from_value(v)) - { - private_detail_te_self.from_value(v); - } - - template - static void - private_detail_te_default_from_value(float, T&& private_detail_te_self, const value& v) - { - from_value_context(private_detail_te_self, v); - } - - template - static auto private_detail_te_default_get_queue(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.get_queue()) - { - return private_detail_te_self.get_queue(); - } - - template - static any_ptr private_detail_te_default_get_queue(float, T&& private_detail_te_self) - { - return get_queue_context(private_detail_te_self); - } - - template - static auto private_detail_te_default_wait_for(char, T&& private_detail_te_self, any_ptr queue) - -> decltype(private_detail_te_self.wait_for(queue)) - { - private_detail_te_self.wait_for(queue); - } - - template - static void private_detail_te_default_wait_for(float, T&& private_detail_te_self, any_ptr queue) - { - wait_for_context(private_detail_te_self, queue); - } - - template - static auto private_detail_te_default_finish_on(char, T&& private_detail_te_self, any_ptr queue) - -> decltype(private_detail_te_self.finish_on(queue)) - { - private_detail_te_self.finish_on(queue); - } - - template - static void - private_detail_te_default_finish_on(float, T&& private_detail_te_self, any_ptr queue) - { - finish_on_context(private_detail_te_self, queue); - } - - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(private_detail_te_default_to_value(char(0), - std::declval()), - private_detail_te_default_from_value(char(0), - std::declval(), - std::declval()), - private_detail_te_default_get_queue(char(0), - std::declval()), - private_detail_te_default_wait_for( - char(0), std::declval(), std::declval()), - private_detail_te_default_finish_on( - char(0), std::declval(), std::declval()), - std::declval().finish(), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - context() = default; - - template , - typename = typename std::enable_if< - not std::is_same, context>{}>::type> - context(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template , - typename = typename std::enable_if< - not std::is_same, context>{}>::type> - context& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - context rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - value to_value() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().to_value(); - } - - void from_value(const value& v) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().from_value(v); - } - - any_ptr get_queue() - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_queue(); - } - - void wait_for(any_ptr queue) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().wait_for(queue); - } - - void finish_on(any_ptr queue) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().finish_on(queue); - } - - void finish() const - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().finish(); - } - - friend bool is_shared(const context& private_detail_x, const context& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual value to_value() const = 0; - virtual void from_value(const value& v) = 0; - virtual any_ptr get_queue() = 0; - virtual void wait_for(any_ptr queue) = 0; - virtual void finish_on(any_ptr queue) = 0; - virtual void finish() const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - value to_value() const override - { - - return private_detail_te_default_to_value(char(0), private_detail_te_value); - } - - void from_value(const value& v) override - { - - private_detail_te_default_from_value(char(0), private_detail_te_value, v); - } - - any_ptr get_queue() override - { - - return private_detail_te_default_get_queue(char(0), private_detail_te_value); - } - - void wait_for(any_ptr queue) override - { - - private_detail_te_default_wait_for(char(0), private_detail_te_value, queue); - } - - void finish_on(any_ptr queue) override - { - - private_detail_te_default_finish_on(char(0), private_detail_te_value, queue); - } - - void finish() const override { private_detail_te_value.finish(); } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const context* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(context* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(context& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const context& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); } - -inline void migraphx_from_value(const value& v, context& ctx) { ctx.from_value(v); } - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/convert_to_json.hpp b/docker/rocm/migraphx/include/migraphx/convert_to_json.hpp deleted file mode 100644 index fbed2df44..000000000 --- a/docker/rocm/migraphx/include/migraphx/convert_to_json.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_CONVERT_TO_JSON_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_CONVERT_TO_JSON_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT std::string convert_to_json(const std::string& str); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/convolution.hpp b/docker/rocm/migraphx/include/migraphx/convolution.hpp deleted file mode 100644 index 6c61e92a9..000000000 --- a/docker/rocm/migraphx/include/migraphx/convolution.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CONVOLUTION_HPP -#define MIGRAPHX_GUARD_RTGLIB_CONVOLUTION_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -void convolution( - Output output, T input, T weights, Padding padding, Stride stride, Dilation dilation, int group) -{ - auto output_shape = output.get_shape(); - auto in_lens = input.get_shape().lens(); - - auto wei_lens = weights.get_shape().lens(); - auto wei_n = wei_lens[0]; - auto wei_c = wei_lens[1]; - std::vector win_size(wei_lens.begin() + 1, wei_lens.end()); - - par_for(output_shape.elements(), [&](auto i) { - auto idx_o = output_shape.multi(i); - auto w = idx_o[1]; - auto n_dim = idx_o.size(); - - std::vector win_start; - for(std::size_t dim = 2; dim < n_dim; ++dim) - { - auto d_2 = dim - 2; - win_start.push_back(std::ptrdiff_t(idx_o[dim] * stride[d_2]) - - std::ptrdiff_t(padding[d_2])); - } - const auto group_id = w / (wei_n / group); - - shape win_shape{output_shape.type(), win_size}; - - double acc = 0.0; - shape_for_each(win_shape, [&](const auto& idx_win) { - auto k = idx_win[0]; - const auto in_ch = group_id * wei_c + k; - std::vector idx(idx_o.begin(), idx_o.end()); - idx[1] = in_ch; - std::vector idx_dil(idx_win.size() - 1); - std::transform(idx_win.cbegin() + 1, - idx_win.cend(), - dilation.cbegin(), - idx_dil.begin(), - [](std::ptrdiff_t ii, std::ptrdiff_t d) { return d * ii; }); - std::transform(idx_dil.begin(), - idx_dil.end(), - win_start.begin(), - idx.begin() + 2, - [](std::ptrdiff_t ii, std::ptrdiff_t jj) { return ii + jj; }); - std::vector idx_wei(idx_o.size()); - idx_wei[0] = w; - std::copy(idx_win.begin(), idx_win.end(), idx_wei.begin() + 1); - if(std::all_of(idx.begin() + 2, idx.end(), [&](auto ii) { return ii >= 0; }) and - std::equal(idx.begin(), - idx.end(), - in_lens.begin(), - in_lens.end(), - std::less{})) - { - acc += input(idx.begin(), idx.end()) * weights(idx_wei.begin(), idx_wei.end()); - } - }); - - output[i] = acc; - }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/copy_assignable_function.hpp b/docker/rocm/migraphx/include/migraphx/copy_assignable_function.hpp deleted file mode 100644 index 04b512028..000000000 --- a/docker/rocm/migraphx/include/migraphx/copy_assignable_function.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_COPY_ASSIGNABLE_FUNCTION_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_COPY_ASSIGNABLE_FUNCTION_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct copy_assignable_function_wrapper -{ - optional f; - - copy_assignable_function_wrapper(F pf) : f(std::move(pf)) {} - copy_assignable_function_wrapper(const copy_assignable_function_wrapper& other) = default; - copy_assignable_function_wrapper(copy_assignable_function_wrapper&& other) noexcept = default; - copy_assignable_function_wrapper& operator=(copy_assignable_function_wrapper other) - { - f.reset(); - if(other.f.has_value()) - f.emplace(std::move(*other.f)); - return *this; - } - - template - auto operator()(Ts&&... xs) const -> decltype((*f)(std::forward(xs)...)) - { - return (*f)(std::forward(xs)...); - } -}; - -template -using copy_assignable_function = - std::conditional_t{}, F, copy_assignable_function_wrapper>; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_COPY_ASSIGNABLE_FUNCTION_HPP diff --git a/docker/rocm/migraphx/include/migraphx/cpp_generator.hpp b/docker/rocm/migraphx/include/migraphx/cpp_generator.hpp deleted file mode 100644 index 9f34ba159..000000000 --- a/docker/rocm/migraphx/include/migraphx/cpp_generator.hpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_CPP_GENERATOR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_CPP_GENERATOR_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct operation; -struct module; -struct shape; - -struct cpp_generator_impl; - -struct MIGRAPHX_EXPORT cpp_generator -{ - using generate_module_callback = std::function&)>; - struct param - { - std::string name; - std::string type; - }; - - struct MIGRAPHX_EXPORT function - { - std::vector params = {}; - std::string body = ""; - std::string return_type = "void"; - std::string name = ""; - std::vector attributes = {}; - std::vector tparams = {}; - function& set_body(const module& m, const generate_module_callback& g); - function& set_body(const std::string& s) - { - body = s; - return *this; - } - function& set_name(const std::string& s) - { - name = s; - return *this; - } - function& set_attributes(std::vector attrs) - { - attributes = std::move(attrs); - return *this; - } - function& set_types(const module& m); - function& set_types(const module& m, const std::function& parse); - function& set_generic_types(const module& m); - function& add_generic_param(const std::string& pname); - function& unused_param(const std::string& pname); - }; - - cpp_generator(); - - // move constructor - cpp_generator(cpp_generator&&) noexcept; - - // copy assignment operator - cpp_generator& operator=(cpp_generator rhs); - - ~cpp_generator() noexcept; - - void fmap(const std::function& f); - - void fresult(const std::function& f); - - void always_return_tuple(bool b = true); - - void add_point_op(const std::string& op_name, const std::string& code); - - std::string generate_point_op(const operation& op, const std::vector& args); - - std::string str() const; - - function generate_module(const module& m, const generate_module_callback& g); - - function generate_module(const module& m); - - std::string create_function(const function& f); - - static std::vector - to_args(const std::vector& inputs, - const std::unordered_map& names); - - private: - std::unique_ptr impl; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_CPP_GENERATOR_HPP diff --git a/docker/rocm/migraphx/include/migraphx/dead_code_elimination.hpp b/docker/rocm/migraphx/include/migraphx/dead_code_elimination.hpp deleted file mode 100644 index 669b27385..000000000 --- a/docker/rocm/migraphx/include/migraphx/dead_code_elimination.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEAD_CODE_ELIMINATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEAD_CODE_ELIMINATION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -struct program; - -/** - * Remove instructions where the output is not used. - */ -struct MIGRAPHX_EXPORT dead_code_elimination -{ - std::string name() const { return "dead_code_elimination"; } - void apply(module& m) const; - void apply(program& p) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/dfor.hpp b/docker/rocm/migraphx/include/migraphx/dfor.hpp deleted file mode 100644 index e40acf3eb..000000000 --- a/docker/rocm/migraphx/include/migraphx/dfor.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_DFOR_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_DFOR_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// Multidimensional for loop -inline auto dfor() -{ - return [](auto f) { f(); }; -} - -template -auto dfor(T x, Ts... xs) -{ - return [=](auto f) { - for(T i = 0; i < x; i++) - { - dfor(xs...)([&](Ts... is) { f(i, is...); }); - } - }; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/dom_info.hpp b/docker/rocm/migraphx/include/migraphx/dom_info.hpp deleted file mode 100644 index a15dcdd4c..000000000 --- a/docker/rocm/migraphx/include/migraphx/dom_info.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DOM_INFO_HPP -#define MIGRAPHX_GUARD_RTGLIB_DOM_INFO_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct MIGRAPHX_EXPORT dominator_info -{ - bool strictly_dominate(instruction_ref ins1, instruction_ref ins2) const; - - std::unordered_map ins2idom; -}; - -MIGRAPHX_EXPORT dominator_info compute_dominator(const module& m); -// MIGRAPHX_EXPORT dominator_info compute_dominator_naive(const module& m); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/dyn_output.hpp b/docker/rocm/migraphx/include/migraphx/dyn_output.hpp deleted file mode 100644 index dd6c00219..000000000 --- a/docker/rocm/migraphx/include/migraphx/dyn_output.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_DYN_OUTPUT_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_DYN_OUTPUT_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct dyn_output -{ - // original shape from the instruction - shape ins_shape; - // shape computed at eval time using input arguments - shape computed_shape; -}; - -/** - * Handle dynamic and static shape at evaluation time. - * If converted to shape type, returns original ins_shape. - * If converted to dyn_output type, will compute an output shape using the input arguments. - */ -template -struct compute_output_shape -{ - F ins_inputs; - - operator dyn_output() const - { - return ins_inputs([](const auto& x, shape ins_shape, const std::vector& inputs) { - if(ins_shape.dynamic()) - return dyn_output{ins_shape, compute_shape(x, to_shapes(inputs))}; - return dyn_output{ins_shape, ins_shape}; - }); - } - - operator shape() const - { - return ins_inputs( - [](const auto&, shape ins_shape, const std::vector&) { return ins_shape; }); - } -}; - -template -compute_output_shape make_compute_output_shape(F f) -{ - return {f}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/include/migraphx/dynamic_loader.hpp b/docker/rocm/migraphx/include/migraphx/dynamic_loader.hpp deleted file mode 100644 index 9fe574fb6..000000000 --- a/docker/rocm/migraphx/include/migraphx/dynamic_loader.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_DYNAMIC_LOADER_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_DYNAMIC_LOADER_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct dynamic_loader_impl; - -struct MIGRAPHX_EXPORT dynamic_loader -{ - template - static fs::path path(T* address) - { - return path(reinterpret_cast(address)); - } - static fs::path path(void* address); - static optional try_load(const fs::path& p); - - dynamic_loader() = default; - - dynamic_loader(const fs::path& p); - - dynamic_loader(const char* image, std::size_t size); - - dynamic_loader(const std::vector& buffer); - - std::shared_ptr get_symbol(const std::string& name) const; - - template - std::function get_function(const std::string& name) const - { - auto s = get_symbol(name); - return [=](auto&&... xs) -> decltype(auto) { - auto f = reinterpret_cast>(s.get()); - return f(std::forward(xs)...); - }; - } - - private: - std::shared_ptr impl; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_DYNAMIC_LOADER_HPP diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_allocation.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_allocation.hpp deleted file mode 100644 index 9929597ab..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_allocation.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_ALLOCATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_ALLOCATION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove memory allocations. This will create a parameter which is the max of all memory used in - * the program. - */ -struct MIGRAPHX_EXPORT eliminate_allocation -{ - std::string allocation_op{}; - std::size_t alignment = 32; - std::string name() const { return "eliminate_allocation"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_common_subexpression.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_common_subexpression.hpp deleted file mode 100644 index 620e967bd..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_common_subexpression.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_COMMON_SUBEXPRESSION_ELIMINATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_COMMON_SUBEXPRESSION_ELIMINATION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove identical instructions. - */ -struct MIGRAPHX_EXPORT eliminate_common_subexpression -{ - std::string name() const { return "eliminate_common_subexpression"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_concat.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_concat.hpp deleted file mode 100644 index a58cbdff4..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_concat.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_CONCAT_HPP -#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_CONCAT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove concat operators by having each operator can write to different chunk of memory. - */ -struct MIGRAPHX_EXPORT eliminate_concat -{ - concat_optimization concat_opt; - std::string name() const { return "eliminate_concat"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_contiguous.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_contiguous.hpp deleted file mode 100644 index 113152205..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_contiguous.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_CONTIGUOUS_HPP -#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_CONTIGUOUS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove contiguous instructions by checking if the operator can use non-standard shapes. - */ -struct MIGRAPHX_EXPORT eliminate_contiguous -{ - std::string op_name; - std::string name() const { return "eliminate_contiguous"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_convert.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_convert.hpp deleted file mode 100644 index fc71a5104..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_convert.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_CONVERTS_HPP -#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_CONVERTS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove nested converts and nop converts. - */ -struct MIGRAPHX_EXPORT eliminate_convert -{ - std::string name() const { return "eliminate_convert"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_data_type.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_data_type.hpp deleted file mode 100644 index 38efd5706..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_data_type.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ELIMINATE_DATA_TYPE_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ELIMINATE_DATA_TYPE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove data types. This will instert convert operators so the data type - * is not used by any operator. - */ -struct MIGRAPHX_EXPORT eliminate_data_type -{ - std::set unsupported_types; - shape::type_t target_type; - std::set unsupported_ops = {"all"}; - std::string name() const { return "eliminate_data_type"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_identity.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_identity.hpp deleted file mode 100644 index c7fdd0023..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_identity.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_IDENTITY_HPP -#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_IDENTITY_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove identity instructions. Currently when used as the last pass, it will - * preserve the semantics of previous program state, therefore dead code elimination - * should not be used afterwards. - */ -struct MIGRAPHX_EXPORT eliminate_identity -{ - std::string name() const { return "eliminate_identity"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/eliminate_pad.hpp b/docker/rocm/migraphx/include/migraphx/eliminate_pad.hpp deleted file mode 100644 index 1f4d5bd86..000000000 --- a/docker/rocm/migraphx/include/migraphx/eliminate_pad.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ELIMINATE_PAD_HPP -#define MIGRAPHX_GUARD_RTGLIB_ELIMINATE_PAD_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Remove pads if they can be written as an - * attribute to another op (im2col, convolution, pooling) - */ -struct MIGRAPHX_EXPORT eliminate_pad -{ - std::string name() const { return "eliminate_pad"; } - - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/env.hpp b/docker/rocm/migraphx/include/migraphx/env.hpp deleted file mode 100644 index f5bdf43df..000000000 --- a/docker/rocm/migraphx/include/migraphx/env.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ENV_HPP -#define MIGRAPHX_GUARD_RTGLIB_ENV_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// Declare a cached environment variable -#define MIGRAPHX_DECLARE_ENV_VAR(x) \ - struct x \ - { \ - static const char* value() { return #x; } \ - }; // NOLINT - -MIGRAPHX_EXPORT bool enabled(const char* name); -MIGRAPHX_EXPORT bool disabled(const char* name); -MIGRAPHX_EXPORT std::vector env(const char* name); - -MIGRAPHX_EXPORT std::size_t value_of(const char* name, std::size_t fallback = 0); - -MIGRAPHX_EXPORT std::string string_value_of(const char* name, std::string fallback = ""); - -template -bool enabled(T) -{ - static const bool result = enabled(T::value()); - return result; -} - -template -bool disabled(T) -{ - static const bool result = disabled(T::value()); - return result; -} - -template -std::size_t value_of(T, std::size_t fallback = 0) -{ - static const std::size_t result = value_of(T::value(), fallback); - return result; -} - -template -std::string string_value_of(T, std::string fallback = "") -{ - static const std::string result = string_value_of(T::value(), fallback); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/erase.hpp b/docker/rocm/migraphx/include/migraphx/erase.hpp deleted file mode 100644 index 4adeccc2d..000000000 --- a/docker/rocm/migraphx/include/migraphx/erase.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_ERASE_HPP -#define MIGRAPHX_GUARD_ERASE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * @brief Erase all elements from a container - * - * @param r The container to erase elements from - * @param value The value to be erased - * @return Returns iterator to erased element - */ -template -auto erase(R&& r, const T& value) -{ - return r.erase(std::remove(r.begin(), r.end(), value), r.end()); -} - -/** - * @brief Erase all elements from a container - * - * @param r The container to erase elements from - * @param pred Predicate function that selects which elements should be erased. - */ -template -void erase_if(R&& r, P&& pred) -{ - auto first = r.begin(); - auto last = r.end(); - while(first != last) - { - if(pred(*first)) - first = r.erase(first); - else - first++; - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/errors.hpp b/docker/rocm/migraphx/include/migraphx/errors.hpp deleted file mode 100644 index aff4cbf23..000000000 --- a/docker/rocm/migraphx/include/migraphx/errors.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_ERRORS_HPP -#define MIGRAPHX_GUARD_ERRORS_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// Represents exceptions that can be thrown by migraphxlib -struct exception : std::runtime_error -{ - unsigned int error; - exception(unsigned int e = 0, const std::string& msg = "") : std::runtime_error(msg), error(e) - { - } -}; - -/** - * @brief Create an exception object - * - * @param context A message that says where the exception occurred - * @param message Custom message for the error - * @return Exceptions - */ -inline exception make_exception(const std::string& context, const std::string& message = "") -{ - return {0, context + ": " + message}; -} - -inline exception -make_exception(const std::string& context, unsigned int e, const std::string& message = "") -{ - return {e, context + ": " + message}; -} - -/** - * @brief Create a message of a file location - * - * @param file The filename - * @param line The line number - * - * @return A string that represents the file location - */ -inline std::string make_source_context(const std::string& file, int line, const std::string& fname) -{ - return file + ":" + std::to_string(line) + ": " + fname; -} - -// NOLINTNEXTLINE -#define MIGRAPHX_MAKE_SOURCE_CTX() migraphx::make_source_context(__FILE__, __LINE__, __func__) - -/** - * @brief Throw an exception with context information - */ -#define MIGRAPHX_THROW(...) throw migraphx::make_exception(MIGRAPHX_MAKE_SOURCE_CTX(), __VA_ARGS__) - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/execution_environment.hpp b/docker/rocm/migraphx/include/migraphx/execution_environment.hpp deleted file mode 100644 index aaeb3e718..000000000 --- a/docker/rocm/migraphx/include/migraphx/execution_environment.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_EXECUTION_ENV_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_EXECUTION_ENV_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct execution_environment -{ - any_ptr queue = any_ptr{}; - bool async = false; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif /* MIGRAPHX_GUARD_MIGRAPHLIB_EXECUTION_ENV_HPP */ diff --git a/docker/rocm/migraphx/include/migraphx/fallthrough.hpp b/docker/rocm/migraphx/include/migraphx/fallthrough.hpp deleted file mode 100644 index 1479dc2a4..000000000 --- a/docker/rocm/migraphx/include/migraphx/fallthrough.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_FALLTHROUGH_HPP -#define MIGRAPHX_GUARD_FALLTHROUGH_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef __clang__ -#define MIGRAPHX_FALLTHROUGH [[clang::fallthrough]] -#else -#define MIGRAPHX_FALLTHROUGH -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/file_buffer.hpp b/docker/rocm/migraphx/include/migraphx/file_buffer.hpp deleted file mode 100644 index ee257cc8f..000000000 --- a/docker/rocm/migraphx/include/migraphx/file_buffer.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_FILE_BUFFER_HPP -#define MIGRAPHX_GUARD_RTGLIB_FILE_BUFFER_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT std::vector -read_buffer(const fs::path& filename, size_t offset = 0, size_t nbytes = 0); -MIGRAPHX_EXPORT std::string read_string(const fs::path& filename); - -MIGRAPHX_EXPORT void write_string(const fs::path& filename, const std::string& buffer); -MIGRAPHX_EXPORT void write_buffer(const fs::path& filename, const char* buffer, std::size_t size); -MIGRAPHX_EXPORT void write_buffer(const fs::path& filename, const std::vector& buffer); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/filesystem.hpp b/docker/rocm/migraphx/include/migraphx/filesystem.hpp deleted file mode 100644 index 555ef18a8..000000000 --- a/docker/rocm/migraphx/include/migraphx/filesystem.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_FILESYSTEM_HPP -#define MIGRAPHX_GUARD_RTGLIB_FILESYSTEM_HPP - -#include - -#if defined(CPPCHECK) -#define MIGRAPHX_HAS_FILESYSTEM 1 -#define MIGRAPHX_HAS_FILESYSTEM_TS 1 -#elif defined(_WIN32) -#if _MSC_VER >= 1920 -#define MIGRAPHX_HAS_FILESYSTEM 1 -#define MIGRAPHX_HAS_FILESYSTEM_TS 0 -#elif _MSC_VER >= 1900 -#define MIGRAPHX_HAS_FILESYSTEM 0 -#define MIGRAPHX_HAS_FILESYSTEM_TS 1 -#else -#define MIGRAPHX_HAS_FILESYSTEM 0 -#define MIGRAPHX_HAS_FILESYSTEM_TS 0 -#endif -#elif defined(__has_include) -#if __has_include() && __cplusplus >= 201703L -#define MIGRAPHX_HAS_FILESYSTEM 1 -#else -#define MIGRAPHX_HAS_FILESYSTEM 0 -#endif -#if __has_include() && __cplusplus >= 201103L -#define MIGRAPHX_HAS_FILESYSTEM_TS 1 -#else -#define MIGRAPHX_HAS_FILESYSTEM_TS 0 -#endif -#else -#define MIGRAPHX_HAS_FILESYSTEM 0 -#define MIGRAPHX_HAS_FILESYSTEM_TS 0 -#endif - -#if MIGRAPHX_HAS_FILESYSTEM -#include -#elif MIGRAPHX_HAS_FILESYSTEM_TS -#include -#else -#error "No filesystem include available" -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#if MIGRAPHX_HAS_FILESYSTEM -namespace fs = ::std::filesystem; -#elif MIGRAPHX_HAS_FILESYSTEM_TS -namespace fs = ::std::experimental::filesystem; -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/fileutils.hpp b/docker/rocm/migraphx/include/migraphx/fileutils.hpp deleted file mode 100644 index 64bf98442..000000000 --- a/docker/rocm/migraphx/include/migraphx/fileutils.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_FILEUTILS_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_FILEUTILS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT fs::path make_executable_filename(std::string_view name); -MIGRAPHX_EXPORT fs::path make_shared_object_filename(std::string_view name); -MIGRAPHX_EXPORT fs::path make_object_file_filename(std::string_view name); -MIGRAPHX_EXPORT fs::path make_static_library_filename(std::string_view name); -MIGRAPHX_EXPORT fs::path append_extension(const fs::path& path, std::string_view ext); - -inline std::string operator+(std::string l, const fs::path& r) { return std::move(l) + r.string(); } - -inline std::string operator+(const fs::path& l, std::string r) { return l.string() + std::move(r); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_MIGRAPHLIB_FILEUTILS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/float8.hpp b/docker/rocm/migraphx/include/migraphx/float8.hpp deleted file mode 100644 index 98a4a7b10..000000000 --- a/docker/rocm/migraphx/include/migraphx/float8.hpp +++ /dev/null @@ -1,452 +0,0 @@ -/* ************************************************************************ - * Copyright (C) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- - * ies of the Software, and to permit persons to whom the Software is furnished - * to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- - * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- - * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ************************************************************************ */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_FLOAT8_HPP -#define MIGRAPHX_GUARD_RTGLIB_FLOAT8_HPP - -// We are clipping/saturation in down conversion by default. Unclipped version is not tested and -// shouldn't be used without having enough tests. -// logic is based on clipping table from here : https://onnx.ai/onnx/technical/float8.html#cast -// NOLINTNEXTLINE -#define MIGRAPHX_F8_DOWNCAST_CLIPPING 1 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace fp8 { - -enum class rounding_mode -{ - standard, // standard rounding is doing RNE -- round to nearest even - stochastic -}; - -enum class f8_type -{ - bf8 = 0, // s1e5m2 - fp8 = 1 // s1e4m3 -}; - -template -class numeric_limits; - -template -struct float8 -{ - uint8_t data = 0x00; - // default constructor - constexpr float8() = default; - // default copy constructor - constexpr float8(const float8& y) = default; - struct from_bits_t - { - }; - static constexpr from_bits_t from_bits() { return from_bits_t(); } - - explicit constexpr float8(uint8_t bits, from_bits_t) : data(bits) {} - - explicit constexpr float8( - float v, - migraphx::fp8::rounding_mode rm = migraphx::fp8::rounding_mode::standard, - uint32_t rng = 0) - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<3, 4, float, FNUZ /*negative_zero_nan*/, true /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#else // MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<3, 4, float, FNUZ /*negative_zero_nan*/, false /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#endif // MIGRAPHX_F8_DOWNCAST_CLIPPING - } - else - { -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<2, 5, float, FNUZ /*negative_zero_nan*/, true /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#else // MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<2, 5, float, FNUZ /*negative_zero_nan*/, false /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#endif // rocblas_F8_downcast_clipping} - } - } - - inline constexpr operator float() const - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - return migraphx::fp8::impl::cast_from_f8<3, 4, float, FNUZ /*negative_zero_nan*/>(data); - } // else - return migraphx::fp8::impl::cast_from_f8<2, 5, float, FNUZ /*negative_zero_nan*/>(data); - } - - inline explicit constexpr operator bool() const { return not is_zero(); } - - inline constexpr bool is_zero() const - { - if constexpr(FNUZ) - { - return data == 0x00; - } - else - { - return (data == 0x00) or (data == 0x80); - } - } - - inline constexpr bool is_nan() const - { - if constexpr(FNUZ) - { - return data == 0x80; - } - else - { - if(T == migraphx::fp8::f8_type::bf8) - { - return (data == 0x7D) or (data == 0x7E) or (data == 0x7F) or (data == 0xFD) or - (data == 0xFE) or (data == 0xFF); - } - else - { - return (data == 0x7F) or (data == 0xFF); - } - } - } - - inline constexpr bool is_inf() const - { - if constexpr(FNUZ) - { - return data == 0x80; - } - else - { - if(T == migraphx::fp8::f8_type::bf8) - { - return (data == 0x7C) or (data == 0xFC); - } - else - { - // no infinities in e4m3fn, represent them as NaNs - return (data == 0x7F) or (data == 0xFF); - } - } - } - -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_UNARY_OP(unary_op, binary_op) \ - constexpr float8& operator unary_op(const float8& rhs) \ - { \ - const auto tmp = static_cast(*this) binary_op static_cast(rhs); \ - *this = static_cast(tmp); \ - return *this; \ - } \ - constexpr float8& operator unary_op(const float& rhs) \ - { \ - const auto tmp = static_cast(*this) binary_op static_cast(rhs); \ - *this = static_cast(tmp); \ - return *this; \ - } - - MIGRAPHX_FP8_UNARY_OP(*=, *) - MIGRAPHX_FP8_UNARY_OP(-=, -) - MIGRAPHX_FP8_UNARY_OP(+=, +) - MIGRAPHX_FP8_UNARY_OP(/=, /) - - inline constexpr float8& operator=(const float8& rhs) = default; - inline constexpr float8& operator=(float8&& rhs) noexcept = default; - - inline constexpr float8& operator=(float rhs) - { - *this = static_cast(rhs); - return *this; - } - - inline constexpr bool operator==(const float8& rhs) const - { - if(rhs.is_nan() or rhs.is_inf() or this->is_nan() or this->is_inf()) - return false; - else if((rhs.is_zero() and this->is_zero()) or (this->data == rhs.data)) - return true; - return false; - } - - inline constexpr bool operator<(const float8& rhs) const - { - const auto we = static_cast(*this); - const auto them = static_cast(rhs); - return we < them; - } - - inline constexpr bool operator>(const float8& rhs) const - { - const auto we = static_cast(*this); - const auto them = static_cast(rhs); - return we > them; - } -}; - -// https://onnx.ai/onnx/technical/float8.html -using fp8e4m3fn = float8; -using fp8e5m2 = float8; -using fp8e4m3fnuz = float8; -using fp8e5m2fnuz = float8; -/* -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_BINARY_OP(binary_op, T, U) \ - inline constexpr U operator binary_op(const T& lhs, const T& rhs) \ - { \ - return U(static_cast(lhs) binary_op static_cast(rhs)); \ - } - -// TODO: these should return floats for binary ops -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_BINARY_OP_GEN_FOR(T) \ - MIGRAPHX_FP8_BINARY_OP(*, T, T) \ - MIGRAPHX_FP8_BINARY_OP(-, T, T) \ - MIGRAPHX_FP8_BINARY_OP(/, T, T) \ - MIGRAPHX_FP8_BINARY_OP(+, T, T) \ - MIGRAPHX_FP8_BINARY_OP(==, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(>=, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(<=, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(>, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(<, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(!=, T, bool) - -MIGRAPHX_FP8_BINARY_OP_GEN_FOR(fp8e5m2) -MIGRAPHX_FP8_BINARY_OP_GEN_FOR(fp8e4m3fn) -MIGRAPHX_FP8_BINARY_OP_GEN_FOR(fp8e5m2fnuz) -MIGRAPHX_FP8_BINARY_OP_GEN_FOR(fp8e4m3fnuz) -*/ - -// Special operator overloading -inline std::ostream& operator<<(std::ostream& os, const fp8e4m3fnuz& rhs) -{ - return os << static_cast(rhs); -} - -inline fp8e4m3fnuz fabs(fp8e4m3fnuz v) -{ - v.data = v.data & 0x7F; // NOLINT - return v; -} - -// Special operator overloading -inline std::ostream& operator<<(std::ostream& os, const fp8e4m3fn& rhs) -{ - return os << static_cast(rhs); -} - -inline fp8e4m3fn fabs(fp8e4m3fn v) -{ - v.data = v.data & 0x7F; // NOLINT - return v; -} - -// Special operator overloading -inline std::ostream& operator<<(std::ostream& os, const fp8e5m2fnuz& rhs) -{ - return os << static_cast(rhs); -} - -inline fp8e5m2fnuz fabs(fp8e5m2fnuz v) -{ - v.data = v.data & 0x7F; // NOLINT - return v; -} -// Special operator overloading -inline std::ostream& operator<<(std::ostream& os, const fp8e5m2& rhs) -{ - return os << static_cast(rhs); -} - -inline fp8e5m2 fabs(fp8e5m2 v) -{ - v.data = v.data & 0x7F; // NOLINT - return v; -} -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = false; - static constexpr fp8e4m3fnuz epsilon() { return fp8e4m3fnuz(0x28, fp8e4m3fnuz::from_bits()); } - // NOLINTNEXTLINE - static constexpr fp8e4m3fnuz quiet_NaN() { return fp8e4m3fnuz(0x80, fp8e4m3fnuz::from_bits()); } - - static constexpr fp8e4m3fnuz max() { return fp8e4m3fnuz(0x7F, fp8e4m3fnuz::from_bits()); } - // this is min value that is not DeNorm. DeNorm min is 0x01 - static constexpr fp8e4m3fnuz min() { return fp8e4m3fnuz(0x08, fp8e4m3fnuz::from_bits()); } - - static constexpr fp8e4m3fnuz lowest() { return fp8e4m3fnuz(0xFF, fp8e4m3fnuz::from_bits()); } -}; - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = false; - static constexpr fp8e4m3fn epsilon() { return fp8e4m3fn(0x20, fp8e4m3fn::from_bits()); } - // NOLINTNEXTLINE - static constexpr fp8e4m3fn quiet_NaN() { return fp8e4m3fn(0x7F, fp8e4m3fn::from_bits()); } - - static constexpr fp8e4m3fn max() { return fp8e4m3fn(0x7E, fp8e4m3fn::from_bits()); } - // this is min value that is not DeNorm. DeNorm min is 0x01 - static constexpr fp8e4m3fn min() { return fp8e4m3fn(0x08, fp8e4m3fn::from_bits()); } - - static constexpr fp8e4m3fn lowest() { return fp8e4m3fn(0xFE, fp8e4m3fn::from_bits()); } -}; - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = false; - static constexpr fp8e5m2fnuz epsilon() { return fp8e5m2fnuz(0x34, fp8e5m2fnuz::from_bits()); } - - static constexpr fp8e5m2fnuz quiet_NaN() // NOLINT - { - return fp8e5m2fnuz(0x80, fp8e5m2fnuz::from_bits()); - } - - static constexpr fp8e5m2fnuz max() { return fp8e5m2fnuz(0x7F, fp8e5m2fnuz::from_bits()); } - // this is min value that is not DeNorm. DeNorm min is 0x01. I am not sure if we want to make - // this distinction. For the floating points we would end up using lowest most of the times. - static constexpr fp8e5m2fnuz min() { return fp8e5m2fnuz(0x4, fp8e5m2fnuz::from_bits()); } - - static constexpr fp8e5m2fnuz lowest() { return fp8e5m2fnuz(0xFF, fp8e5m2fnuz::from_bits()); } -}; - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = true; - static constexpr fp8e5m2 epsilon() { return fp8e5m2(0x34, fp8e5m2::from_bits()); } - // 7D, 7E, 7F are positive NaNs and FD, FE, FF are negative NaNs - static constexpr fp8e5m2 quiet_NaN() { return fp8e5m2(0xFF, fp8e5m2::from_bits()); } // NOLINT - - static constexpr fp8e5m2 max() { return fp8e5m2(0x7B, fp8e5m2::from_bits()); } - // this is min value that is not DeNorm. DeNorm min is 0x01. I am not sure if we want to make - // this distinction. For the floating points we would end up using lowest most of the times. - static constexpr fp8e5m2 min() { return fp8e5m2(0x4, fp8e5m2::from_bits()); } - - static constexpr fp8e5m2 lowest() { return fp8e5m2(0xFB, fp8e5m2::from_bits()); } - // 7C and FC both are infinity - static constexpr fp8e5m2 infinity() { return fp8e5m2(0x7C, fp8e5m2::from_bits()); } -}; -} // namespace fp8 -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -// ================================================================================================= -// define numeric limits for the new data type -// NOLINTBEGIN(cert-dcl58-cpp) -namespace std { - -template -inline bool isfinite(migraphx::fp8::float8 x) -{ - return not x.is_inf() and not x.is_nan(); -} - -template -inline bool isnan(migraphx::fp8::float8 x) -{ - return x.is_nan(); -} - -template -class numeric_limits> - : public migraphx::fp8::numeric_limits> -{ -}; -template -struct common_type, U> : std::common_type -{ -}; -template -struct common_type> : std::common_type -{ -}; -template -struct common_type, migraphx::fp8::float8> -{ - using type = migraphx::fp8::float8; -}; - -template -struct common_type, migraphx::fp8::float8> -{ - using type = float; -}; - -template -struct common_type, - migraphx::fp8::float8> -{ - using type = float; -}; - -template -struct common_type, - migraphx::generic_float> -{ - using type = float; -}; - -template -struct common_type, migraphx::fp8::float8> - : std::common_type -{ -}; - -template -struct common_type, migraphx::generic_float> - : std::common_type -{ -}; - -} // namespace std -// NOLINTEND(cert-dcl58-cpp) -// ================================================================================================= -#endif // MIGRAPHX_GUARD_RTGLIB_FLOAT8_HPP diff --git a/docker/rocm/migraphx/include/migraphx/float8_impl.hpp b/docker/rocm/migraphx/include/migraphx/float8_impl.hpp deleted file mode 100644 index 8316b123a..000000000 --- a/docker/rocm/migraphx/include/migraphx/float8_impl.hpp +++ /dev/null @@ -1,328 +0,0 @@ -/* ************************************************************************ - * Copyright (C) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- - * ies of the Software, and to permit persons to whom the Software is furnished - * to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- - * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- - * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ************************************************************************ */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_FLOAT8_IMPL_HPP -#define MIGRAPHX_GUARD_RTGLIB_FLOAT8_IMPL_HPP -#include -#include -#include -#include -#include -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace fp8 { -namespace impl { - -// NOLINTBEGIN -template -constexpr uint8_t cast_to_f8(T f_x, bool stoch = false, uint32_t rng = 0) -{ - constexpr bool is_float = std::is_same::value; - // half is not supported for now - constexpr bool is_half = false; - static_assert(Wm + We == 7, "Wm+We==7"); - static_assert(is_float or is_half, "Only float can be cast to f8"); - - const uint32_t mfmt = (sizeof(T) == 4) ? 23 : 10; - typename std::conditional::type x; - - if constexpr(sizeof(T) == 4) - x = migraphx::bit_cast(f_x); - else - x = migraphx::bit_cast(f_x); - - uint32_t head = 0; - uint32_t mantissa = 0; - int exponent = 0; - uint32_t bias = 0; - uint32_t sign = 0; - if constexpr(sizeof(T) == 4) - { - head = x & 0xFF800000; - mantissa = x & 0x7FFFFF; - exponent = (head >> 23) & 0xFF; - sign = head >> 31; - bias = 127; - } - else - { - head = x & 0xFC00; - mantissa = x & 0x3FF; - exponent = (head >> 10) & 0x1F; - sign = head >> 15; - bias = 15; - } - - uint32_t signed_inf = (sign << 7) + (((1 << We) - 1) << Wm); - uint32_t signed_all_ones = (sign << 7) + ((((1 << We) - 1) << Wm) + ((1 << Wm) - 1)); - - // Calcualte maximum singed value FLT_MAX, FLT_MIN - uint32_t signed_max = signed_all_ones; - if(not NegativeZeroNan) - signed_max = (Wm == 2) ? (signed_max - 4) : (signed_max - 1); - - // Deal with inf and NaNs - if(NegativeZeroNan) // For the FNUZ cases, it is simple just return NaNs - { - if((sizeof(T) == 4 and ((x & 0x7F800000) == 0x7F800000)) or - (sizeof(T) == 2 and ((x & 0x7C00) == 0x7C00))) - return 0x80; - } - else - { - // calculate most common NaN mantissa for FP8, which is all Ones in binary - uint32_t nan_mantissa = 1; - for(auto i = 1; i < Wm; ++i) - { - nan_mantissa |= (nan_mantissa << 1); - } - if((sizeof(T) == 4 and ((x & 0x7F800000) == 0x7F800000)) or - (sizeof(T) == 2 and ((x & 0x7C00) == 0x7C00))) - { - // infinity - if(mantissa == 0) - { - if(sign == 0) - return (Wm == 2) ? 0x7B : 0x7E; - else - return (Wm == 2) ? 0xFB : 0xFE; - } - else // NaNs - return signed_inf + nan_mantissa; - } - } - // handle positive zero - if(x == 0) - return 0; - // handle negative zero - else if((sizeof(T) == 4 and x == 0x80000000) or (sizeof(T) == 2 and x == 0x8000)) - { - return NegativeZeroNan ? 0 : 0x80; // For FNUZ types neg zero is just positive zero - } - - /* First need to check if it is normal or denorm as there is a difference of implict 1 - Then need to adjust the exponent to align with the F8 exponent, in the meanwhile, shift - The mantissa. Then for stochastic rounding, add rng to mantissa and truncate. And for - RNE, no need to add rng. Then probably need to check whether there is carry and adjust - exponent and mantissa again*/ - - // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent bits - const int f8_bias = (1 << (We - 1u)) - 1 + (NegativeZeroNan ? 1 : 0); - const int f8_denormal_act_exponent = 1 - f8_bias; // actual exponent of f8 denormal - /* act_exponent is the actual exponent of fp32/fp16 (after subtracting bias) - f8_exponent is the converted f8 exponent with bias encoding - exponent_diff is the diff between fp32/fp16 exponent and f8 exponent, - the difference needs to be adjusted and mantissa shifted*/ - int act_exponent = 0; - int f8_exponent = 0; - int exponent_diff = 0; - - if(exponent == 0 and mantissa != 0) - { // fp32/fp16 is in denormal. - /* fp32 denormal is below 2^-127 so it is usually not a concern here, we mostly concern fp16 - here. In this case, f8 is usually in denormal. But there could be exceptions. fp16 denormal - has exponent bias 15 while bf8 with FNUZ has exponent bias 16. It means that there are some - numbers in fp16 denormal but they are bf8 (FNUZ) normals - smallest bf8 (FNUZ) normal is - 2^-15. fp16 numbers where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 - are bf8 (FNUZ) normal. In this case, the fp16 mantissa should be shift left by 1 */ - act_exponent = 1 - bias; - exponent_diff = f8_denormal_act_exponent - - act_exponent; // actual exponent is exponent-bias+1 as it is denormal - } - else - { // fp32/fp16 is normal with implicit 1 - act_exponent = exponent - bias; - if(act_exponent <= f8_denormal_act_exponent) - { - /* This is the case where fp32/fp16 is normal but it is in f8 denormal range. - For example fp8 FNUZ mode, denormal exponent is -7, but if the fp32/fp16 - actual exponent is -7, it is actually larger due to the implict 1, - Therefore it needs to be adjust to -6 and mantissa shift right by 1. - So for fp32/fp16, exponent -8 is the cut point to convert to fp8 FNUZ */ - exponent_diff = f8_denormal_act_exponent - act_exponent; - } - else - { // both fp32/fp16 and f8 are in normal range - exponent_diff = - 0; // exponent_diff=0 does not mean there is no difference for this case, - // act_exponent could be larger. Just that it does not need shift mantissa - } - mantissa += (1u << mfmt); // Add the implicit 1 into mantissa - } - - // need to know whether the number is right in the middle of two adjacent fp8 numbers. use max - // value of 31 to avoid undefined behaviour - bool midpoint = (mantissa & ((1u << std::min(31u, mfmt - Wm + exponent_diff)) - 1)) == - (1u << std::min(31u, mfmt - Wm + exponent_diff - 1)); - /* This part is a bit tricky. The judgment of whether it is a tie needs to be done before we - shift right as shift right could rip off some residual part and make something not midpoint look - like midpoint. For example, the fp16 number 0x1002 (0 00100 0000000010), it is larger than - midpoint, but after shift right by 4 bits, it would look like midpoint. - */ - - if(exponent_diff > 0) - mantissa >>= std::min(31u, uint32_t(exponent_diff)); - else if(exponent_diff == -1) - mantissa <<= -exponent_diff; - bool implicit_one = mantissa & (1 << mfmt); - // if there is no implict 1, it means the f8 is denormal and need to adjust to denorm exponent - f8_exponent = - (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias - (implicit_one ? 0 : 1); - - // Now we have the exponent and mantissa adjusted - uint32_t drop_mask = (1u << (mfmt - Wm)) - 1; - bool odd = - mantissa & (1u << (mfmt - Wm)); // if the least significant bit that is not truncated is 1 - /* - This part is doing rounding by adding mantissa part that is going to get dropped. - e.g. if the dropped part for less than 0.5 than it would round down. - if the dropped part is more than 0.5 then it would round up by rolling carry to LSB of retained - mantissa. - For the mid point when bit pattern is like this for Odd: `xy1:10000000` for Odd and - `xy0:10000000` for the Even. where `:` is delimiter for dropped v/s retained part. - For the odd case : - this will add xy1:10000000 + 000:10000000 which would roll over carry to LSB of retained - part making it RNE. - For the even case : this will add xy0:10000000 + 000:01111111 which would - round down and keep number Even - */ - mantissa += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask; - - // Now we deal with overflow - if(f8_exponent == 0 and ((1 << mfmt) & mantissa)) - { - f8_exponent = 1; // denormal overflow to become normal, promote exponent - } - else if((1 << (mfmt + 1)) & mantissa) - { - mantissa >>= 1; - f8_exponent++; - } - - mantissa >>= (mfmt - Wm); - - // above range: quantize to maximum possible float of the same sign - // for e5m2 case, max_exp is 14, since exp = 15 is reserved for Infs and Nans - const int max_exp = (1 << We) - ((NegativeZeroNan or Wm == 3) ? 1 : 2); - if(f8_exponent > max_exp) - { - if(Clip) - return signed_max; - else - { - // https://onnx.ai/onnx/technical/float8.html#cast - if(NegativeZeroNan) - return 0x80; - else - return (Wm == 2) ? signed_inf : signed_all_ones; - } - } - - if(f8_exponent == 0 and mantissa == 0) - return NegativeZeroNan ? 0 : (sign << 7); - mantissa &= (1 << Wm) - 1; - return (sign << 7) | (f8_exponent << Wm) | mantissa; -} -// NOLINTEND - -template -constexpr T cast_from_f8(uint8_t x) -{ - // half is not supported for now - constexpr bool is_half = false; - constexpr bool is_float = std::is_same::value; - static_assert(is_float or is_half, "Only float are supported"); - - constexpr int weo = is_half ? 5 : 8; - constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7); - // NOLINTNEXTLINE - T f_inf, f_neg_inf, f_nan, f_neg0; - - if constexpr(is_float) - { - const uint32_t if_inf = 0x7F800000; - const uint32_t if_neg_inf = 0xFF800000; - const uint32_t if_nan = 0x7F800001; - const uint32_t if_neg0 = 0x80000000; - f_inf = migraphx::bit_cast(if_inf); - f_neg_inf = migraphx::bit_cast(if_neg_inf); - f_nan = migraphx::bit_cast(if_nan); - f_neg0 = migraphx::bit_cast(if_neg0); - } - - if(x == 0) - return 0; - - uint32_t sign = x >> 7; // NOLINT - uint32_t mantissa = x & ((1 << Wm) - 1); // NOLINT - int exponent = (x & 0x7F) >> Wm; // NOLINT - if(NegativeZeroNan) - { - if(x == 0x80) - return f_nan; - } - else - { - if(x == 0x80) - return f_neg0; - if(exponent == ((1 << We) - 1) and Wm == 2) // NOLINT - return (mantissa == 0) ? (sign ? f_neg_inf : f_inf) : f_nan; - else if(Wm == 3 and (x == 0x7F or x == 0xFF)) - return f_nan; - } - typename std::conditional::type retval; - - const int exp_low_cutoff = - (1 << (weo - 1)) - (1 << (We - 1)) + 1 - (NegativeZeroNan ? 1 : 0); // NOLINT - - // subnormal input - if(exponent == 0) - { - // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above - int sh = 1 + __builtin_clz(mantissa) - (32 - Wm); - mantissa <<= sh; // NOLINT - exponent += 1 - sh; - mantissa &= ((1 << Wm) - 1); // NOLINT - } - exponent += exp_low_cutoff - 1; - mantissa <<= wmo - Wm; // NOLINT - - // subnormal output (occurs when T=half, We=5, negative_zero_nan=true) - if(exponent <= 0) - { - mantissa |= 1 << wmo; // NOLINT - mantissa >>= 1 - exponent; // NOLINT - exponent = 0; - } - - if(sizeof(T) == 2) - retval = (sign << 15) | (exponent << 10) | mantissa; // NOLINT - else - retval = (sign << 31) | (exponent << 23) | mantissa; // NOLINT - return migraphx::bit_cast(retval); -} - -} // namespace impl -} // namespace fp8 -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_RTGLIB_FLOAT8_IMPL diff --git a/docker/rocm/migraphx/include/migraphx/float_equal.hpp b/docker/rocm/migraphx/include/migraphx/float_equal.hpp deleted file mode 100644 index c473ac8c1..000000000 --- a/docker/rocm/migraphx/include/migraphx/float_equal.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_FLOAT_EQUAL_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_FLOAT_EQUAL_HPP - -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -using common_type = typename std::common_type::type; - -struct float_equal_fn -{ - template {})> - static bool apply(T x, T y) - { - return std::isfinite(x) and std::isfinite(y) and - std::nextafter(x, std::numeric_limits::lowest()) <= y and - std::nextafter(x, std::numeric_limits::max()) >= y; - } - - template {})> - static bool apply(T x, T y) - { - return x == y; - } - - template - bool operator()(T x, U y) const - { - return float_equal_fn::apply>(x, y); - } -}; - -static constexpr float_equal_fn float_equal{}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/fp8_ocp_to_fnuz.hpp b/docker/rocm/migraphx/include/migraphx/fp8_ocp_to_fnuz.hpp deleted file mode 100644 index 19e4a1cda..000000000 --- a/docker/rocm/migraphx/include/migraphx/fp8_ocp_to_fnuz.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_FP8_OCP_TO_FNUZ_HPP -#define MIGRAPHX_GUARD_RTGLIB_FP8_OCP_TO_FNUZ_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Convert fp8e4m3fn to fp8e4m3fnuz for hardware that only supports fp8e4m3fnuz data types - * intrinsically. Conversion uses the same bit representation and adjusts scaling factors at the - * dequantization. Using the same bit representation from fp8e4m3fn to fp8e4m3fnuz halves the - * floating point representation. This pass should run before simplify_qdq so that the scales and - * zero points calculated by simplify_qdq have the correct adjusted scaling factors - */ -struct MIGRAPHX_EXPORT fp8_ocp_to_fnuz -{ - std::string name() const { return "fp8_ocp_to_fnuz"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/fp8_types.hpp b/docker/rocm/migraphx/include/migraphx/fp8_types.hpp deleted file mode 100644 index d0ea85a6e..000000000 --- a/docker/rocm/migraphx/include/migraphx/fp8_types.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_FP8_TYPES_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_FP8_TYPES_HPP -#include -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct fp8_types -{ - const std::set types = {shape::fp8e4m3fnuz_type, - shape::fp8e5m2fnuz_type, - shape::fp8e4m3fn_type, - shape::fp8e5m2_type}; - - std::set get() const { return types; } -}; -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_FP8_TYPES_HPP diff --git a/docker/rocm/migraphx/include/migraphx/fp_to_double.hpp b/docker/rocm/migraphx/include/migraphx/fp_to_double.hpp deleted file mode 100644 index 875e76c31..000000000 --- a/docker/rocm/migraphx/include/migraphx/fp_to_double.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_FP_TO_DOUBLE_HPP -#define MIGRAPHX_GUARD_RTGLIB_FP_TO_DOUBLE_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Convert floating point values to double precision. - */ -struct MIGRAPHX_EXPORT fp_to_double -{ - std::set convert_fp_types = {shape::type_t::half_type, - shape::type_t::float_type}; - std::string name() const { return "fp_to_double"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/functional.hpp b/docker/rocm/migraphx/include/migraphx/functional.hpp deleted file mode 100644 index f7a79c423..000000000 --- a/docker/rocm/migraphx/include/migraphx/functional.hpp +++ /dev/null @@ -1,271 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_FUNCTIONAL_HPP -#define MIGRAPHX_GUARD_RTGLIB_FUNCTIONAL_HPP - -#include -#include - -// Similiar to decltype(auto) except it will propagate any substitution failures -// NOLINTNEXTLINE -#define MIGRAPHX_RETURNS(...) \ - ->decltype(__VA_ARGS__) { return __VA_ARGS__; } - -// Lifts an expression into a function object so it can be passed to a higher-order function -// NOLINTNEXTLINE -#define MIGRAPHX_LIFT(...) \ - [](auto&&... private_lifts_xs) MIGRAPHX_RETURNS( \ - (__VA_ARGS__)(static_cast(private_lifts_xs)...)) - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct swallow -{ - template - constexpr swallow(Ts&&...) - { - } -}; - -template -auto tuple_size(const T&) -{ - return typename std::tuple_size::type{}; -} - -namespace detail { - -template -struct fix_f -{ - F f; - - template - R operator()(Ts&&... xs) const - { - return f(*this, std::forward(xs)...); - } -}; - -template -struct seq -{ - using type = seq; -}; - -template -struct merge_seq; - -template -struct merge_seq, seq> : seq -{ -}; - -template -struct gens : merge_seq::type, typename gens::type> -{ -}; - -template <> -struct gens<0> : seq<> -{ -}; -template <> -struct gens<1> : seq<0> -{ -}; - -template -constexpr void repeat_c_impl(F f, seq) -{ - swallow{(f(std::integral_constant{}), 0)...}; -} - -template -constexpr auto sequence_c_impl(F&& f, seq) -{ - return f(std::integral_constant{}...); -} - -} // namespace detail - -template -constexpr void repeat_c(F f) -{ - detail::repeat_c_impl(f, detail::gens{}); -} - -template -constexpr auto sequence_c(F&& f) -{ - return detail::sequence_c_impl(f, detail::gens{}); -} - -template -constexpr auto sequence(IntegerConstant ic, F&& f) -{ - return sequence_c(f); -} - -template -constexpr void each_args(F f, Ts&&... xs) -{ - swallow{(f(std::forward(xs)), 0)...}; -} - -template -constexpr void each_args(F) -{ -} - -template -auto unpack(F f, T&& x) -{ - return sequence(tuple_size(x), [&](auto... is) { f(std::get(static_cast(x))...); }); -} - -/// Implements a fix-point combinator -template -detail::fix_f fix(F f) -{ - return {f}; -} - -template -auto fix(F f) -{ - return fix(f); -} - -template -auto fold_impl(F&&, T&& x) -{ - return std::forward(x); -} - -template -auto fold_impl(F&& f, T&& x, U&& y, Ts&&... xs) -{ - return fold_impl(f, f(std::forward(x), std::forward(y)), std::forward(xs)...); -} - -template -auto fold(F f) -{ - return [=](auto&&... xs) { return fold_impl(f, std::forward(xs)...); }; -} - -template -auto pack(Ts... xs) -{ - return [=](auto f) { return f(xs...); }; -} - -inline auto pack_join() { return pack(); } - -template -auto pack_join(Ps... ps) -{ - return fold([](auto p1, auto p2) { - return p1([=](auto... xs) { return p2([=](auto... ys) { return pack(xs..., ys...); }); }); - })(ps...); -} - -template -auto by(F f, Proj proj) -{ - return [=](auto&&... xs) { return f(proj(std::forward(xs))...); }; -} - -template -auto index_of(T& x) -{ - return [&](auto&& y) { return x[y]; }; -} - -template -decltype(auto) front_args(T&& x, Ts&&...) -{ - return static_cast(x); -} - -template -decltype(auto) back_args(Ts&&... xs) -{ - return std::get(std::tuple(static_cast(xs)...)); -} - -template -auto pop_front_args(T&&, Ts&&... xs) -{ - return [&](auto f) { f(static_cast(xs)...); }; -} - -template -auto pop_back_args(Ts&&... xs) -{ - return [&](auto f) { - using tuple_type = std::tuple; - auto t = tuple_type(static_cast(xs)...); - return sequence_c( - [&](auto... is) { return f(std::get(static_cast(t))...); }); - }; -} - -template -struct always_f -{ - T x; - template - constexpr T operator()(Ts&&...) const - { - return x; - } -}; - -template -auto always(T x) -{ - return always_f{x}; -} - -struct id -{ - template - constexpr T operator()(T&& x) const - { - return static_cast(x); - } -}; - -template -void nop(Ts&&...) -{ -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/fuse_concat.hpp b/docker/rocm/migraphx/include/migraphx/fuse_concat.hpp deleted file mode 100644 index 40f538145..000000000 --- a/docker/rocm/migraphx/include/migraphx/fuse_concat.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_FUSE_CONCAT_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_FUSE_CONCAT_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -struct MIGRAPHX_EXPORT fuse_concat -{ - std::string name() const { return "fuse_concat"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_FUSE_CONCAT_HPP diff --git a/docker/rocm/migraphx/include/migraphx/fuse_pointwise.hpp b/docker/rocm/migraphx/include/migraphx/fuse_pointwise.hpp deleted file mode 100644 index 1fdb2d2b5..000000000 --- a/docker/rocm/migraphx/include/migraphx/fuse_pointwise.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -struct MIGRAPHX_EXPORT fuse_pointwise -{ - std::string name() const { return "fuse_pointwise"; } - void apply(module_pass_manager& mpm) const; - - bool enable_rewrite_reshapes = true; - bool enable_rewrite_broadcasts = false; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP diff --git a/docker/rocm/migraphx/include/migraphx/fuse_pointwise_reduce.hpp b/docker/rocm/migraphx/include/migraphx/fuse_pointwise_reduce.hpp deleted file mode 100644 index 63d78d236..000000000 --- a/docker/rocm/migraphx/include/migraphx/fuse_pointwise_reduce.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_REDUCE_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_REDUCE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -struct MIGRAPHX_EXPORT fuse_pointwise_reduce -{ - std::size_t split_size = 32768; - std::string name() const { return "fuse_pointwise_reduce"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_REDUCE_HPP diff --git a/docker/rocm/migraphx/include/migraphx/fuse_reduce.hpp b/docker/rocm/migraphx/include/migraphx/fuse_reduce.hpp deleted file mode 100644 index c5176eda8..000000000 --- a/docker/rocm/migraphx/include/migraphx/fuse_reduce.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_FUSE_REDUCE_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_FUSE_REDUCE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -struct MIGRAPHX_EXPORT fuse_reduce -{ - std::string name() const { return "fuse_reduce"; } - void apply(module_pass_manager& mpm) const; - - bool enable_rewrite_reshapes = true; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_FUSE_POINTWISE_HPP diff --git a/docker/rocm/migraphx/include/migraphx/gemm.hpp b/docker/rocm/migraphx/include/migraphx/gemm.hpp deleted file mode 100644 index 912930c1a..000000000 --- a/docker/rocm/migraphx/include/migraphx/gemm.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GEMM_HPP -#define MIGRAPHX_GUARD_RTGLIB_GEMM_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -void gemm(tensor_view cmat, tensor_view amat, tensor_view bmat, F alpha, F beta) -{ - std::size_t n_dims = cmat.get_shape().lens().size(); - std::size_t dim_0 = n_dims - 2; - std::size_t dim_1 = n_dims - 1; - auto k = amat.get_shape().lens()[dim_1]; - - assert(amat.get_shape().lens()[dim_1] == bmat.get_shape().lens()[dim_0]); - assert(cmat.get_shape().lens()[dim_0] == amat.get_shape().lens()[dim_0]); - assert(cmat.get_shape().lens()[dim_1] == bmat.get_shape().lens()[dim_1]); - auto cs = cmat.get_shape(); - - par_for(cs.elements(), [&](auto i) { - auto c_idx = cs.multi(i); - auto a_idx = c_idx; - auto b_idx = c_idx; - double s = 0.0; - dfor(k)([&](auto kk) { - a_idx[dim_1] = b_idx[dim_0] = kk; - s += static_cast(amat(a_idx.begin(), a_idx.end())) * - static_cast(bmat(b_idx.begin(), b_idx.end())); - }); - cmat(c_idx.begin(), c_idx.end()) = alpha * s + cmat(c_idx.begin(), c_idx.end()) * beta; - }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/generate.hpp b/docker/rocm/migraphx/include/migraphx/generate.hpp deleted file mode 100644 index c30e4d16a..000000000 --- a/docker/rocm/migraphx/include/migraphx/generate.hpp +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_GENERATE_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_GENERATE_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -enum class random_mode -{ - legacy, - random -}; - -template {})> -constexpr T normalize(unsigned long z, random_mode m) -{ - auto max = (m == random_mode::legacy) ? 32 : 1ULL << (sizeof(T) * 8 - 1); - const double range = max / 2.0; - double result = -1.0 + (z % max) / range; - // Expected output: between -1.0 and 1.0 - return T(result); -} - -template {} and not is_floating_point{})> -constexpr T normalize(unsigned long z, random_mode m) -{ - const long long max = - (m == random_mode::legacy) ? 1ULL << (sizeof(T) * 5) : 1ULL << (sizeof(T) * 6 - 1); - const auto half_max = max / 2; - auto result = half_max - (z % max); - // Expected output: between -half_max and half_max - return T(result); -} - -template {} and std::is_integral{} and - not std::is_same{})> -constexpr T normalize(unsigned long z, random_mode m) -{ - const auto max = - (m == random_mode::legacy) ? 1ULL << (sizeof(T) * 5) : 1ULL << (sizeof(T) * 8 - 1); - // Expected output: between 0 and max - 1 - return z % max; -} - -template {})> -constexpr bool normalize(unsigned long z, random_mode) -{ - // Expected output: 0 or 1b - return static_cast(z % 2); -} - -template -struct xorshf96_generator -{ - unsigned long x = 123456789; - unsigned long y = 362436069; - unsigned long z; - random_mode mode; - - xorshf96_generator(unsigned long seed, random_mode m) : z(521288629ULL ^ seed), mode(m) {} - - constexpr T operator()() noexcept - { - x ^= x << 16U; - x ^= x >> 5U; - x ^= x << 1U; - - unsigned long t = x; - x = y; - y = z; - z = t ^ x ^ y; - - return normalize(z, mode); - } -}; - -template -struct xorshift_generator -{ - unsigned long x; - random_mode mode; - - xorshift_generator(unsigned long seed, random_mode m) : x(521288629ULL ^ seed), mode(m) {} - - constexpr T operator()() noexcept - { - x ^= x >> 12U; - x ^= x << 25U; - x ^= x >> 27U; - return normalize(x * 0x2545F4914F6CDD1D, mode); - } -}; - -template -auto generate_tensor_data(const migraphx::shape& s, - unsigned long seed, - random_mode m = random_mode::legacy) -{ - auto result = make_shared_array(s.element_space()); - std::generate(result.get(), result.get() + s.element_space(), xorshf96_generator{seed, m}); - return result; -} - -template -auto fill_tensor_data(const migraphx::shape& s, double value = 0) -{ - auto result = make_shared_array(s.element_space()); - std::generate(result.get(), result.get() + s.element_space(), [=] { return value; }); - return result; -} - -MIGRAPHX_EXPORT argument fill_argument(shape s, double value = 0); - -MIGRAPHX_EXPORT argument generate_argument(shape s, - unsigned long seed = 0, - random_mode m = random_mode::legacy); - -MIGRAPHX_EXPORT literal generate_literal(shape s, unsigned long seed = 0); - -MIGRAPHX_EXPORT literal abs(literal l); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/generic_float.hpp b/docker/rocm/migraphx/include/migraphx/generic_float.hpp deleted file mode 100644 index c886bc035..000000000 --- a/docker/rocm/migraphx/include/migraphx/generic_float.hpp +++ /dev/null @@ -1,476 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_MIGRAPHX_GENERIC_FLOAT_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_GENERIC_FLOAT_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -constexpr unsigned int all_ones() noexcept -{ - return (1u << N) - 1u; -} - -template -constexpr int countl_zero(T value) -{ - unsigned int r = 0; - for(; value != 0u; value >>= 1u) - r++; - return 8 * sizeof(value) - r; -} - -constexpr std::size_t bit_ceil(std::size_t v) -{ - if(v <= 1) - return 1; - v--; - v |= v >> 1u; - v |= v >> 2u; - v |= v >> 4u; - v |= v >> 8u; - v |= v >> 16u; - v |= v >> 32u; - return v + 1; -} - -constexpr std::size_t integer_divide_ceil(std::size_t x, std::size_t y) -{ - return (x + y - std::size_t{1}) / y; -} - -template -struct unsigned_type -{ -}; - -template <> -struct unsigned_type<1> -{ - using type = std::uint8_t; -}; - -template <> -struct unsigned_type<2> -{ - using type = std::uint16_t; -}; - -template <> -struct unsigned_type<4> -{ - using type = std::uint32_t; -}; - -template <> -struct unsigned_type<8> -{ - using type = std::uint64_t; -}; - -struct float32_parts -{ - unsigned int mantissa : 23; - unsigned int exponent : 8; - unsigned int sign : 1; - - static constexpr unsigned int exponent_width() { return 8; } - - static constexpr unsigned int mantissa_width() { return 23; } - - static constexpr unsigned int max_exponent() { return all_ones<8>(); } - - static constexpr int exponent_bias() { return all_ones<7>(); } - - constexpr float to_float() const noexcept { return migraphx::bit_cast(*this); } -}; - -constexpr float32_parts get_parts(float f) { return migraphx::bit_cast(f); } - -template -struct __attribute__((packed, may_alias)) generic_float -{ - using type = typename unsigned_type::type; - - type mantissa : MantissaSize; - type exponent : ExponentSize; - type sign : 1; - - static constexpr int exponent_bias() { return all_ones(); } - - explicit constexpr generic_float(float f = 0.0) noexcept { from_float(get_parts(f)); } - - constexpr generic_float& operator=(float f) noexcept - { - from_float(get_parts(f)); - return *this; - } - - constexpr generic_float operator-() const noexcept - { - generic_float result = *this; - result.sign = not this->sign; - return result; - } - - constexpr generic_float operator+() const noexcept { return *this; } - - constexpr float to_float() const noexcept - { - float32_parts f{}; - f.sign = sign; - - if(exponent == 0 and ExponentSize != float32_parts::exponent_width()) // subnormal fps - { - - if(mantissa == 0) - { - f.exponent = 0; - f.mantissa = 0; - } - else - { - type shift = 0; - f.mantissa = mantissa; - - if(MantissaSize < float32_parts::mantissa_width()) - { - shift = MantissaSize - ((sizeof(type) * 8) - countl_zero(mantissa)); - f.mantissa <<= (shift + 1u); - } - - f.exponent = float32_parts::exponent_bias() - exponent_bias() - shift; - f.mantissa = f.mantissa << (float32_parts::mantissa_width() - MantissaSize); - } - } - else if(exponent == all_ones()) - { - f.mantissa = mantissa << (float32_parts::mantissa_width() - MantissaSize); - f.exponent = float32_parts::max_exponent(); - } - else - { - f.mantissa = mantissa << (float32_parts::mantissa_width() - MantissaSize); - constexpr const int diff = float32_parts::exponent_bias() - exponent_bias(); - f.exponent = int(exponent) + diff; - } - - return f.to_float(); - } - - constexpr void from_float(float32_parts f) noexcept - { - sign = f.sign; - - if(f.exponent == 0) - { - exponent = 0; - mantissa = f.mantissa >> (float32_parts::mantissa_width() - MantissaSize); - } - else if(f.exponent == float32_parts::max_exponent()) - { - exponent = all_ones(); - mantissa = f.mantissa >> (float32_parts::mantissa_width() - MantissaSize); - } - else - { - constexpr const int diff = float32_parts::exponent_bias() - exponent_bias(); - auto e = int(f.exponent) - diff; - - if(e >= static_cast(all_ones())) - { - exponent = all_ones(); - mantissa = 0; - } - else if(e < 1) - { - exponent = 0; - - auto shift = diff - int(f.exponent); - auto shift_amount = shift + (float32_parts::mantissa_width() - MantissaSize) + 1; - - if(shift_amount < (sizeof(unsigned int) * 8)) - { - mantissa = (f.mantissa | (1u << float32_parts::mantissa_width())) >> - (shift + (float32_parts::mantissa_width() - MantissaSize) + 1); - } - else - { - mantissa = 0; - } - } - else - { - exponent = int(f.exponent) - diff; - mantissa = f.mantissa >> (float32_parts::mantissa_width() - MantissaSize); - } - } - - exponent = std::min(exponent, all_ones()); - } - - constexpr bool is_normal() const noexcept - { - return exponent != all_ones() and exponent != 0; - } - - constexpr bool is_inf() const noexcept - { - return exponent == all_ones() and mantissa == 0; - } - - constexpr bool is_nan() const noexcept - { - return exponent == all_ones() and mantissa != 0; - } - - constexpr bool is_finite() const noexcept { return exponent != all_ones(); } - - constexpr operator float() const noexcept { return this->to_float(); } - - static constexpr generic_float infinity() - { - generic_float x{}; - x.exponent = all_ones(); - return x; - } - - static constexpr generic_float snan() - { - generic_float x{}; - x.exponent = all_ones(); - x.mantissa = 1u << (MantissaSize - 2u); - return x; - } - - static constexpr generic_float qnan() - { - generic_float x{}; - x.exponent = all_ones(); - x.mantissa = 1u << (MantissaSize - 1u); - return x; - } - - static constexpr generic_float min() - { - generic_float x{}; - x.exponent = 1; - x.mantissa = 0; - return x; - } - - static constexpr generic_float denorm_min() - { - generic_float x{}; - x.exponent = 0; - x.mantissa = 1; - x.sign = 0; - return x; - } - - static constexpr generic_float lowest() - { - generic_float x{}; - x.exponent = all_ones() - 1; - x.mantissa = all_ones(); - x.sign = 1; - return x; - } - - static constexpr generic_float max() - { - generic_float x{}; - x.exponent = all_ones() - 1; - x.mantissa = all_ones(); - x.sign = 0; - return x; - } - - static constexpr generic_float epsilon() - { - generic_float x{1.0}; - x.mantissa++; - return generic_float{x.to_float() - 1.0f}; - } -// NOLINTNEXTLINE -#define MIGRAPHX_GENERIC_FLOAT_ASSIGN_OP(op) \ - constexpr generic_float& operator op(const generic_float & rhs) \ - { \ - float self = *this; \ - float frhs = rhs; \ - self op frhs; \ - *this = generic_float(self); \ - return *this; \ - } - MIGRAPHX_GENERIC_FLOAT_ASSIGN_OP(*=) - MIGRAPHX_GENERIC_FLOAT_ASSIGN_OP(-=) - MIGRAPHX_GENERIC_FLOAT_ASSIGN_OP(+=) - MIGRAPHX_GENERIC_FLOAT_ASSIGN_OP(/=) -// NOLINTNEXTLINE -#define MIGRAPHX_GENERIC_FLOAT_BINARY_OP(op) \ - friend constexpr generic_float operator op(const generic_float& x, const generic_float& y) \ - { \ - return generic_float(float(x) op float(y)); \ - } - MIGRAPHX_GENERIC_FLOAT_BINARY_OP(*) - MIGRAPHX_GENERIC_FLOAT_BINARY_OP(-) - MIGRAPHX_GENERIC_FLOAT_BINARY_OP(+) - MIGRAPHX_GENERIC_FLOAT_BINARY_OP(/) -// NOLINTNEXTLINE -#define MIGRAPHX_GENERIC_FLOAT_COMPARE_OP(op) \ - friend constexpr bool operator op(const generic_float& x, const generic_float& y) \ - { \ - return float(x) op float(y); \ - } - MIGRAPHX_GENERIC_FLOAT_COMPARE_OP(<) - MIGRAPHX_GENERIC_FLOAT_COMPARE_OP(<=) - MIGRAPHX_GENERIC_FLOAT_COMPARE_OP(>) - MIGRAPHX_GENERIC_FLOAT_COMPARE_OP(>=) - - friend constexpr bool operator==(const generic_float& x, const generic_float& y) - { - if(not x.is_finite() or not y.is_finite()) - return false; - - if((x.mantissa == 0 and x.exponent == 0) and (y.mantissa == 0 and y.exponent == 0)) - { - return true; - } - - return std::tie(x.mantissa, x.exponent, x.sign) == std::tie(y.mantissa, y.exponent, y.sign); - } - - friend constexpr bool operator!=(const generic_float& x, const generic_float& y) - { - return not(x == y); - } - - constexpr generic_float& operator++() noexcept - { - *this += generic_float(1.0f); - return *this; - } - - const generic_float operator++(int) noexcept // NOLINT(readability-const-return-type) - { - generic_float temp = *this; - *this += generic_float(1.0f); - return temp; - } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -// NOLINTBEGIN(cert-dcl58-cpp) -namespace std { - -template -class numeric_limits> -{ - public: - static constexpr bool has_infinity = true; - static constexpr migraphx::generic_float epsilon() - { - return migraphx::generic_float::epsilon(); - } - - static constexpr migraphx::generic_float quiet_NaN() - { - return migraphx::generic_float::qnan(); - } - - static constexpr migraphx::generic_float signaling_NaN() - { - return migraphx::generic_float::snan(); - } - - static constexpr migraphx::generic_float max() - { - return migraphx::generic_float::max(); - } - - static constexpr migraphx::generic_float min() - { - return migraphx::generic_float::min(); - } - - static constexpr migraphx::generic_float lowest() - { - return migraphx::generic_float::lowest(); - } - - static constexpr migraphx::generic_float infinity() - { - return migraphx::generic_float::infinity(); - } - - static constexpr migraphx::generic_float denorm_min() - { - return migraphx::generic_float::denorm_min(); - } -}; - -template -struct common_type, T> : std::common_type -{ -}; - -template -struct common_type> : std::common_type -{ -}; - -template -struct common_type, migraphx::generic_float> -{ - using type = migraphx::generic_float; -}; - -template -struct common_type, migraphx::generic_float> -{ - using type = float; -}; - -} // namespace std -// NOLINTEND(cert-dcl58-cpp) - -#endif // MIGRAPHX_GUARD_MIGRAPHX_GENERIC_FLOAT_HPP diff --git a/docker/rocm/migraphx/include/migraphx/half.hpp b/docker/rocm/migraphx/include/migraphx/half.hpp deleted file mode 100644 index b92942557..000000000 --- a/docker/rocm/migraphx/include/migraphx/half.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_HALF_HPP -#define MIGRAPHX_GUARD_RTGLIB_HALF_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using half = migraphx::generic_float<10, 5>; - -namespace detail { -template -struct deduce -{ - using type = T; -}; -} // namespace detail - -template -using deduce = typename detail::deduce::type; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/hash.hpp b/docker/rocm/migraphx/include/migraphx/hash.hpp deleted file mode 100644 index e82c36394..000000000 --- a/docker/rocm/migraphx/include/migraphx/hash.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_HASH_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_HASH_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -std::size_t hash_value(const T& v) -{ - return std::hash{}(v); -} - -template -void hash_combine(std::size_t& seed, const T& v) -{ - seed ^= hash_value(v) + 0x9e3779b9 + (seed << 6u) + (seed >> 2u); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_HASH_HPP diff --git a/docker/rocm/migraphx/include/migraphx/inline_module.hpp b/docker/rocm/migraphx/include/migraphx/inline_module.hpp deleted file mode 100644 index dd6336033..000000000 --- a/docker/rocm/migraphx/include/migraphx/inline_module.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_INLINE_MODULE_HPP -#define MIGRAPHX_GUARD_RTGLIB_INLINE_MODULE_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct MIGRAPHX_EXPORT inline_module -{ - std::string name() const { return "inline_module"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/insert_pad.hpp b/docker/rocm/migraphx/include/migraphx/insert_pad.hpp deleted file mode 100644 index 3f940d0e3..000000000 --- a/docker/rocm/migraphx/include/migraphx/insert_pad.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_INSERT_PAD_HPP -#define MIGRAPHX_GUARD_RTGLIB_INSERT_PAD_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * insert pads if attribute of padding is asymmetrical - */ -struct MIGRAPHX_EXPORT insert_pad -{ - std::unordered_set ops = {"convolution", "pooling", "im2col"}; - std::string name() const { return "insert_pad"; } - - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/instruction.hpp b/docker/rocm/migraphx/include/migraphx/instruction.hpp deleted file mode 100644 index 5ff3a83eb..000000000 --- a/docker/rocm/migraphx/include/migraphx/instruction.hpp +++ /dev/null @@ -1,189 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_INSTRUCTION_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_INSTRUCTION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT shape compute_shape(const operation& op, const std::vector& args); -MIGRAPHX_EXPORT shape compute_shape(const operation& op, - const std::vector& args, - const std::vector& mods); -MIGRAPHX_EXPORT std::vector to_shapes(const std::vector& args); -MIGRAPHX_EXPORT std::vector try_compute_shape(const operation& op, - const std::vector& inputs); - -MIGRAPHX_EXPORT bool reaches(instruction_ref start, instruction_ref end); - -struct MIGRAPHX_EXPORT instruction -{ - instruction() {} - - instruction(operation o, shape r, std::vector args); - - instruction(operation o, - shape r, - std::vector args, - std::vector modules); - - instruction(literal l); - - void replace(operation o); - - void recompute_shape(); - - void clear_arguments(); - - MIGRAPHX_EXPORT friend bool operator==(const instruction& i, instruction_ref ref); - - bool valid(instruction_ref start, bool check_order = false) const; - - bool valid() const; - - shape get_shape() const; - const literal& get_literal() const; - - const operation& get_operator() const; - - std::string name() const; - - const std::vector& inputs() const; - - const std::vector& module_inputs() const; - - /// Where this instruction is used as an input to another instruction - const std::vector& outputs() const; - - MIGRAPHX_EXPORT friend bool operator==(const instruction& x, const instruction& y); - - MIGRAPHX_EXPORT friend bool operator!=(const instruction& x, const instruction& y); - - MIGRAPHX_EXPORT friend bool operator==(instruction_ref ref, const instruction& i); - - MIGRAPHX_EXPORT friend bool operator!=(const instruction& i, instruction_ref ref); - - MIGRAPHX_EXPORT friend bool operator!=(instruction_ref ref, const instruction& i); - - void add_output(instruction_ref ins); - - template - void remove_output(const T& ins) - { - migraphx::erase(output, ins); - } - - static void replace_refs(instruction_ref ins, - const std::unordered_map& map_insts, - const std::unordered_map& map_mods); - - static void backreference(instruction_ref ref); - - static void replace_argument(instruction_ref ins, instruction_ref old, instruction_ref new_ins); - - static void replace_mod_argument(instruction_ref ins, module_ref old, module_ref new_mod); - - static void - replace(instruction_ref ins, operation o, const shape& r, std::vector args); - - static void replace(instruction_ref ins, - operation o, - const shape& r, - std::vector args, - std::vector module_args); - - bool can_eval() const; - - bool is_undefined() const; - - argument eval(bool check_eval = true) const; - - void finalize(context& ctx); - - static instruction_ref get_output_alias(instruction_ref ins, bool shallow = false); - - void set_normalized(bool value = true); - bool is_normalized() const; - - bool need_normalization() const; - - operation normalized_operator() const; - - std::size_t get_target_id() const; - - void set_target_id(std::size_t tid); - - void debug_print() const; - - static void print(std::ostream& os, - instruction_ref ins, - const std::unordered_map& names); - - private: - // internal - void replace(operation o, const shape& r, std::vector args); - - // internal - void replace(operation o, - const shape& r, - std::vector args, - std::vector mdl_args); - - // internal - void replace(std::vector args); - - // internal - void replace(std::vector args, std::vector mdl_args); - - // internal - void replace_argument(instruction_ref old, instruction_ref new_ins); - - // internal - void replace_mod_argument(module_ref old, module_ref new_mod); - - void replace(const shape& r); - - operation op; - shape result{}; - std::vector output; - std::vector arguments; - std::vector module_args; - literal lit; - bool normalized = false; - std::size_t target_id = 0; -}; -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/instruction_ref.hpp b/docker/rocm/migraphx/include/migraphx/instruction_ref.hpp deleted file mode 100644 index 7845860e4..000000000 --- a/docker/rocm/migraphx/include/migraphx/instruction_ref.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_INSTRUCTION_REF_HPP -#define MIGRAPHX_GUARD_INSTRUCTION_REF_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct instruction; -#if defined(_WIN32) && !defined(NDEBUG) && !defined(CPPCHECK) -struct instruction_ref : std::list::iterator -{ - using instruction_iter = std::list::iterator; - using instruction_const_iter = std::list::const_iterator; - - instruction_ref() = default; - instruction_ref(const instruction_iter& other) : instruction_iter(other) {} - - template {} or - std::is_same{})> - friend bool operator==(const T& x, const U& y) - { - return x._Unwrapped()._Ptr == y._Unwrapped()._Ptr; - } - - template {} or - std::is_same{})> - friend bool operator!=(const T& x, const U& y) - { - return not(x == y); - } -}; -#else -using instruction_ref = std::list::iterator; -#endif - -MIGRAPHX_EXPORT migraphx::instruction* as_address(const instruction_ref& ins) noexcept; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -namespace std { -template <> -struct hash // NOLINT -{ - using argument_type = migraphx::instruction_ref; - using result_type = std::size_t; - result_type operator()(const migraphx::instruction_ref& x) const noexcept - { - return std::hash{}(migraphx::as_address(x)); - } -}; - -template <> -struct equal_to // NOLINT -{ - using argument_type = migraphx::instruction_ref; - using result_type = bool; - result_type operator()(const migraphx::instruction_ref& x, - const migraphx::instruction_ref& y) const noexcept - { - return migraphx::as_address(x) == migraphx::as_address(y); - } -}; - -} // namespace std - -#ifdef _MSC_VER -#include -#endif - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/iota_iterator.hpp b/docker/rocm/migraphx/include/migraphx/iota_iterator.hpp deleted file mode 100644 index fe07d0e46..000000000 --- a/docker/rocm/migraphx/include/migraphx/iota_iterator.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_IOTA_ITERATOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_IOTA_ITERATOR_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct basic_iota_iterator -{ - Iterator index; - F f; - - using difference_type = std::ptrdiff_t; - using reference = decltype(f(std::declval())); - using value_type = typename std::remove_reference::type; - using pointer = typename std::add_pointer::type; - using iterator_category = std::random_access_iterator_tag; - - basic_iota_iterator& operator+=(int n) - { - index += n; - return *this; - } - - basic_iota_iterator& operator-=(int n) - { - index -= n; - return *this; - } - - basic_iota_iterator& operator++() - { - index++; - return *this; - } - - basic_iota_iterator& operator--() - { - index--; - return *this; - } - - basic_iota_iterator operator++(int) // NOLINT - { - basic_iota_iterator it = *this; - index++; - return it; - } - - basic_iota_iterator operator--(int) // NOLINT - { - basic_iota_iterator it = *this; - index--; - return it; - } - reference operator*() const { return f(index); } - pointer operator->() const { return &f(index); } - reference operator[](int n) const { return f(index + n); } -}; - -template -inline basic_iota_iterator make_basic_iota_iterator(T x, F f) -{ - return basic_iota_iterator{x, f}; -} - -template -inline basic_iota_iterator operator+(basic_iota_iterator x, - std::ptrdiff_t y) -{ - return x += y; -} - -template -inline basic_iota_iterator operator+(std::ptrdiff_t x, - basic_iota_iterator y) -{ - return y + x; -} - -template -inline std::ptrdiff_t operator-(basic_iota_iterator x, - basic_iota_iterator y) -{ - return x.index - y.index; -} - -template -inline basic_iota_iterator operator-(basic_iota_iterator x, - std::ptrdiff_t y) -{ - return x -= y; -} - -template -inline bool operator==(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index == y.index; -} - -template -inline bool operator!=(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index != y.index; -} - -template -inline bool operator<(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index < y.index; -} - -template -inline bool operator>(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index > y.index; -} - -template -inline bool operator>=(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index >= y.index; -} - -template -inline bool operator<=(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index <= y.index; -} - -using iota_iterator = basic_iota_iterator; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/iterator.hpp b/docker/rocm/migraphx/include/migraphx/iterator.hpp deleted file mode 100644 index 372a07ef6..000000000 --- a/docker/rocm/migraphx/include/migraphx/iterator.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_ITERATOR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_ITERATOR_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -auto is_end(rank<2>, Iterator it, EndIterator) -> decltype(not it._M_dereferenceable()) -{ - return not it._M_dereferenceable(); -} - -template -auto is_end(rank<1>, Iterator it, EndIterator last) -{ - return it == last; -} - -template -bool is_end(Iterator it, EndIterator last) -{ - return is_end(rank<2>{}, it, last); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_ITERATOR_HPP diff --git a/docker/rocm/migraphx/include/migraphx/iterator_for.hpp b/docker/rocm/migraphx/include/migraphx/iterator_for.hpp deleted file mode 100644 index 3a53cc02d..000000000 --- a/docker/rocm/migraphx/include/migraphx/iterator_for.hpp +++ /dev/null @@ -1,123 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ITERATOR_FOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_ITERATOR_FOR_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct iterator_for_select -{ - template - static T deref(T x) - { - return x; - } - - template - static auto begin(T* x) - { - return x->begin(); - } - - template - static auto end(T* x) - { - return x->end(); - } -}; - -struct iterator_for_select_reverse -{ - template - static auto deref(T x) - { - return std::prev(x.base()); - } - - template - static auto begin(T* x) - { - return std::make_reverse_iterator(x->end()); - } - - template - static auto end(T* x) - { - return std::make_reverse_iterator(x->begin()); - } -}; - -template -struct iterator_for_range -{ - T* base; - using base_iterator = std::remove_reference_t; - - struct iterator - { - using difference_type = std::ptrdiff_t; - using reference = decltype(std::declval()); - using value_type = std::remove_reference_t; - using pointer = std::add_pointer_t; - using iterator_category = std::input_iterator_tag; - base_iterator i; - auto operator*() const { return Selector::deref(i); } - base_iterator operator++() { return ++i; } - bool operator==(const iterator& rhs) const { return i == rhs.i; } - bool operator!=(const iterator& rhs) const { return i != rhs.i; } - }; - - iterator begin() const - { - assert(base != nullptr); - return {Selector::begin(base)}; - } - iterator end() const - { - assert(base != nullptr); - return {Selector::end(base)}; - } -}; -template -iterator_for_range iterator_for(T& x) -{ - return {&x}; -} - -template -iterator_for_range reverse_iterator_for(T& x) -{ - return {&x}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/json.hpp b/docker/rocm/migraphx/include/migraphx/json.hpp deleted file mode 100644 index e4225cce6..000000000 --- a/docker/rocm/migraphx/include/migraphx/json.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_JSON_HPP -#define MIGRAPHX_GUARD_RTGLIB_JSON_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT std::string to_pretty_json_string(const value& val, std::size_t indent = 4); -MIGRAPHX_EXPORT std::string to_json_string(const value& val); -MIGRAPHX_EXPORT value from_json_string(const std::string& str); -MIGRAPHX_EXPORT value from_json_string(const char* str, std::size_t size); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/layout_convolution.hpp b/docker/rocm/migraphx/include/migraphx/layout_convolution.hpp deleted file mode 100644 index 9e45033a8..000000000 --- a/docker/rocm/migraphx/include/migraphx/layout_convolution.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_LAYOUT_CONVOLUTION_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_LAYOUT_CONVOLUTION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -/** - * Transform convolutions layout - */ -struct MIGRAPHX_EXPORT layout_convolution -{ - bool channels_last = false; - std::string name() const { return "layout_convolution"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_LAYOUT_CONVOLUTION_HPP diff --git a/docker/rocm/migraphx/include/migraphx/lexing.hpp b/docker/rocm/migraphx/include/migraphx/lexing.hpp deleted file mode 100644 index e3c802da7..000000000 --- a/docker/rocm/migraphx/include/migraphx/lexing.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_LEXING_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_LEXING_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using lexer = std::function; - -template -inline auto lex_while(P p) -{ - return [=](const char* start, const char* end) { - return std::find_if(start, end, [&](char c) { return not p(c); }); - }; -} - -template -inline auto lex_if(P p) -{ - return [=](const char* start, const char*) { - if(p(*start)) - return start + 1; - return start; - }; -} - -MIGRAPHX_EXPORT std::function -lex_equal(const std::string& s); - -MIGRAPHX_EXPORT std::vector -tokenize(const char* start, const char* end, const std::vector& lexers); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/lifetime.hpp b/docker/rocm/migraphx/include/migraphx/lifetime.hpp deleted file mode 100644 index 02a4694cb..000000000 --- a/docker/rocm/migraphx/include/migraphx/lifetime.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_LIFETIME_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_LIFETIME_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -enum class lifetime -{ - local, - global, - borrow -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_LIFETIME_HPP diff --git a/docker/rocm/migraphx/include/migraphx/literal.hpp b/docker/rocm/migraphx/include/migraphx/literal.hpp deleted file mode 100644 index 2990fadc1..000000000 --- a/docker/rocm/migraphx/include/migraphx/literal.hpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_LITERAL_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_LITERAL_HPP - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * @brief Represents a raw literal - * @details This stores the literal has a raw buffer that is owned by this class - */ -struct literal : raw_data -{ - literal() {} - - /*! - * Empty literal with a specific shape type - */ - explicit literal(shape::type_t shape_type) : m_shape(shape_type, {}) {} - - template , shape::type_t ShapeType = shape::get_type{}> - literal(U x) : buffer(make_shared_array(sizeof(T))), m_shape(ShapeType) - { - static_assert(std::is_trivially_copyable{}, "Literals can only be trivial types"); - *(reinterpret_cast(buffer.get())) = x; - } - - template - literal(const shape& s, const std::vector& x) - : buffer(make_shared_array(s.bytes())), m_shape(s) - { - static_assert(std::is_trivially_copyable{}, "Literals can only be trivial types"); - fill(x.begin(), x.end()); - } - - template - literal(const shape& s, const std::initializer_list& x) - : buffer(make_shared_array(s.bytes())), m_shape(s) - { - static_assert(std::is_trivially_copyable{}, "Literals can only be trivial types"); - fill(x.begin(), x.end()); - } - - template - literal(const shape& s, Iterator start, Iterator end) - : buffer(make_shared_array(s.bytes())), m_shape(s) - { - fill(start, end); - } - - // Directly copies buffer of x - template - literal(const shape& s, T* x) : buffer(make_shared_array(s.bytes())), m_shape(s) - { - std::copy(x, x + s.bytes(), buffer.get()); - } - - /// Whether data is available - bool empty() const { return this->buffer == nullptr; } - - /// Provides a raw pointer to the data - const char* data() const { return this->buffer.get(); } - - const shape& get_shape() const { return this->m_shape; } - - std::vector get_sub_objects() const { return {}; } - - /// Convert the data to an argument - argument get_argument() const - { - auto b = make_shared_array(buffer.get(), buffer.get() + m_shape.bytes()); - return {m_shape, [b]() { return b.get(); }}; - } - - private: - std::shared_ptr buffer; - shape m_shape; - - // Keeps the same data ordering as the given container - template - void fill(Iterator start, Iterator end) - { - assert(std::distance(start, end) == m_shape.elements()); - m_shape.visit_type([&](auto as) { - auto output = make_view(m_shape, as.from(buffer.get())); - std::copy(start, end, output.begin()); - }); - } -}; - -template -literal transform(literal l, F f) -{ - literal result; - l.visit([&](auto x) { - using type = std::remove_cv_t; - std::vector output(x.size(), type(0)); - std::transform(x.begin(), x.end(), output.begin(), f); - result = literal{l.get_shape(), output}; - }); - return result; -} - -template -literal transform(literal l1, literal l2, F f) -{ - assert(l1.get_shape() == l2.get_shape()); - literal result; - visit_all(l1, l2)([&](auto x, auto y) { - using type = std::remove_cv_t; - std::vector output(x.size(), type(0)); - std::transform(x.begin(), x.end(), y.begin(), output.begin(), f); - result = literal{l1.get_shape(), output}; - }); - return result; -} - -MIGRAPHX_EXPORT void migraphx_to_value(value& v, const literal& l); -MIGRAPHX_EXPORT void migraphx_from_value(const value& v, literal& l); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/liveness.hpp b/docker/rocm/migraphx/include/migraphx/liveness.hpp deleted file mode 100644 index 6d9715a8a..000000000 --- a/docker/rocm/migraphx/include/migraphx/liveness.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_LIVENESS_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_LIVENESS_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// This will do liveness analysis on the module, and it will call the -// function `f` with the instruction and the set of the other instructions -// that are live -template -void liveness(const module& m, F f) -{ - auto implicit_deps = m.calc_implicit_deps(); - std::unordered_set live_set; - auto rp = reverse(m); - for(auto rins : iterator_for(rp)) // NOLINT - { - // The base iterator is one ahead, so we need to use the previous iterator - auto ins = std::prev(rins.base()); - // Add live variables - auto add_live_variables = [&](const auto& inputs) { - for(auto input : inputs) - { - auto i = instruction::get_output_alias(input); - // Skip if variable comes from parent - if(not m.has_instruction(i)) - continue; - live_set.insert(i); - } - }; - add_live_variables(ins->inputs()); - add_live_variables(implicit_deps[ins]); - // Remove last usage - auto it = live_set.find(ins); - if(it != live_set.end()) - { - live_set.erase(it); - f(ins, live_set); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_LIVENESS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/load_save.hpp b/docker/rocm/migraphx/include/migraphx/load_save.hpp deleted file mode 100644 index 19b467bf6..000000000 --- a/docker/rocm/migraphx/include/migraphx/load_save.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_LOAD_SAVE_HPP -#define MIGRAPHX_GUARD_RTGLIB_LOAD_SAVE_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct file_options -{ - std::string format = "msgpack"; -}; - -MIGRAPHX_EXPORT program load(const std::string& filename, - const file_options& options = file_options{}); -MIGRAPHX_EXPORT program load_buffer(const std::vector& buffer, - const file_options& options = file_options{}); -MIGRAPHX_EXPORT program load_buffer(const char* buffer, - std::size_t size, - const file_options& options = file_options{}); - -MIGRAPHX_EXPORT void -save(const program& p, const std::string& filename, const file_options& options = file_options{}); -MIGRAPHX_EXPORT std::vector save_buffer(const program& p, - const file_options& options = file_options{}); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/make_op.hpp b/docker/rocm/migraphx/include/migraphx/make_op.hpp deleted file mode 100644 index 19907b94d..000000000 --- a/docker/rocm/migraphx/include/migraphx/make_op.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MAKE_OP_HPP -#define MIGRAPHX_GUARD_RTGLIB_MAKE_OP_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT operation make_op(const std::string& name); -MIGRAPHX_EXPORT operation make_op(const std::string& name, - const std::initializer_list>& v); -MIGRAPHX_EXPORT operation make_op_from_value(const std::string& name, const value& v); - -// A template overload is added for migraphx::value so the initializer_list -// cannot be passed in directly. This is to enforce at compile-time that all -// initializer_list are key-value pairs, whereas migraphx::value allows other -// types of initializer_list such as for arrays. -template -operation make_op(const std::string& name, const Value& v) -{ - return make_op_from_value(name, v); -} - -MIGRAPHX_EXPORT operation make_json_op(const std::string& name, const std::string& s); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/make_shared_array.hpp b/docker/rocm/migraphx/include/migraphx/make_shared_array.hpp deleted file mode 100644 index e3d807a26..000000000 --- a/docker/rocm/migraphx/include/migraphx/make_shared_array.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_MAKE_SHARED_ARRAY_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_MAKE_SHARED_ARRAY_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -std::shared_ptr make_shared_array(size_t size) -{ - // cppcheck-suppress migraphx-UseSmartPointer - return std::shared_ptr(new T[size](), std::default_delete()); // NOLINT -} - -template -std::shared_ptr make_shared_array(Iterator start, Iterator last) -{ - auto result = make_shared_array(std::distance(start, last)); - std::copy(start, last, result.get()); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/make_signed.hpp b/docker/rocm/migraphx/include/migraphx/make_signed.hpp deleted file mode 100644 index 8aa080ef5..000000000 --- a/docker/rocm/migraphx/include/migraphx/make_signed.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MAKE_SIGNED_HPP -#define MIGRAPHX_GUARD_RTGLIB_MAKE_SIGNED_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -typename std::conditional_t{}, std::make_signed, std::enable_if>:: - type - make_signed(T x) -{ - return x; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/manage_ptr.hpp b/docker/rocm/migraphx/include/migraphx/manage_ptr.hpp deleted file mode 100644 index a098d9fd1..000000000 --- a/docker/rocm/migraphx/include/migraphx/manage_ptr.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_MANAGE_PTR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_MANAGE_PTR_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template // NOLINT -struct manage_deleter -{ - template - void operator()(T* x) const - { - if(x != nullptr) - { - (void)f(x); - } - } -}; - -struct null_deleter -{ - template - void operator()(T*) const - { - } -}; - -template // NOLINT -using manage_ptr = std::unique_ptr>; - -template -struct element_type -{ - using type = typename T::element_type; -}; - -template -using remove_ptr = typename std:: - conditional_t{}, std::remove_pointer, element_type>::type; - -template -using shared = std::shared_ptr>; - -template -shared share(T p) -{ - return shared{std::move(p)}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#define MIGRAPHX_MANAGE_PTR(T, F) \ - migraphx::manage_ptr, decltype(&F), &F> // NOLINT - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/marker.hpp b/docker/rocm/migraphx/include/migraphx/marker.hpp deleted file mode 100644 index 8800c3e1b..000000000 --- a/docker/rocm/migraphx/include/migraphx/marker.hpp +++ /dev/null @@ -1,317 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MARKER_HPP -#define MIGRAPHX_GUARD_MARKER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef DOXYGEN - -/// Marker is an interface to general marking functions, such as rocTX markers. - -#else - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT marker -{ - // - void mark_start(instruction_ref ins_ref); - // - void mark_start(const program& prog); - // - void mark_stop(instruction_ref ins); - // - void mark_stop(const program& prog); -}; - -#else - -struct marker -{ - private: - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().mark_start( - std::declval()), - std::declval().mark_start( - std::declval()), - std::declval().mark_stop( - std::declval()), - std::declval().mark_stop(std::declval()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - marker() = default; - - template , - typename = typename std::enable_if< - not std::is_same, marker>{}>::type> - marker(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template , - typename = typename std::enable_if< - not std::is_same, marker>{}>::type> - marker& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - marker rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - void mark_start(instruction_ref ins_ref) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().mark_start(ins_ref); - } - - void mark_start(const program& prog) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().mark_start(prog); - } - - void mark_stop(instruction_ref ins) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().mark_stop(ins); - } - - void mark_stop(const program& prog) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().mark_stop(prog); - } - - friend bool is_shared(const marker& private_detail_x, const marker& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual void mark_start(instruction_ref ins_ref) = 0; - virtual void mark_start(const program& prog) = 0; - virtual void mark_stop(instruction_ref ins) = 0; - virtual void mark_stop(const program& prog) = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - void mark_start(instruction_ref ins_ref) override - { - - private_detail_te_value.mark_start(ins_ref); - } - - void mark_start(const program& prog) override { private_detail_te_value.mark_start(prog); } - - void mark_stop(instruction_ref ins) override { private_detail_te_value.mark_stop(ins); } - - void mark_stop(const program& prog) override { private_detail_te_value.mark_stop(prog); } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const marker* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(marker* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(marker& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const marker& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/match/dq_helpers.hpp b/docker/rocm/migraphx/include/migraphx/match/dq_helpers.hpp deleted file mode 100644 index cdb40ae97..000000000 --- a/docker/rocm/migraphx/include/migraphx/match/dq_helpers.hpp +++ /dev/null @@ -1,62 +0,0 @@ - -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MATCH_DQ_HELPERS_HPP -#define MIGRAPHX_GUARD_MATCH_DQ_HELPERS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace match { - -/** - * Find dequantizelinear (DQ) instruction with constant scale and zero point input - * while skipping broadcast instructions between DQ and scale/zero point. Used - * in simplify_qdq and fp8_ocp_to_fnuz. - */ -inline auto dequantizelinear_op(const std::string& scale, const std::string& zp) -{ - return match::name("dequantizelinear")( - match::arg(1)(match::skip_broadcasts(match::is_constant().bind(scale))), - match::arg(2)(match::skip_broadcasts(match::is_constant().bind(zp)))); -} - -/** - * Skip certain operators after DQ instruction. - * Used in simplify_qdq and fp8_ocp_to_fnuz. - */ -template -auto skip_post_dq_ops(Ms... ms) -{ - return match::skip(match::name( - "broadcast", "multibroadcast", "contiguous", "transpose", "reshape", "convert"))(ms...); -} - -} // namespace match -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/match/gelu_erf.hpp b/docker/rocm/migraphx/include/migraphx/match/gelu_erf.hpp deleted file mode 100644 index c83abcc56..000000000 --- a/docker/rocm/migraphx/include/migraphx/match/gelu_erf.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MATCH_GELU_ERF_HPP -#define MIGRAPHX_GUARD_MATCH_GELU_ERF_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace match { - -namespace detail { -template -struct gelu_erf_matcher -{ - F f; - auto erf_fn() const - { - auto mul_1_sqrt_2 = f("mul")( - either_arg(0, 1)(none_of(has_value(M_SQRT1_2)).bind("x"), has_value(M_SQRT1_2))); - auto div_sqrt_2 = f("div")(args(none_of(has_value(M_SQRT2)).bind("x"), has_value(M_SQRT2))); - return f("erf")(used_once(), arg(0)(used_once(), any_of(mul_1_sqrt_2, div_sqrt_2))); - } - - auto add_erf() const - { - return f("add")(used_once(), either_arg(0, 1)(erf_fn(), has_value(1.0))); - } - - auto one_half() const { return has_value(0.5); } - - auto matcher() const { return unordered_tree(f("mul"), one_half(), add_erf(), any()); } -}; -} // namespace detail - -template -auto gelu_erf(F f) -{ - return detail::gelu_erf_matcher{f}.matcher(); -} - -inline auto gelu_erf() -{ - return gelu_erf([](auto x) { return name(x); }); -} - -} // namespace match -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MATCH_GELU_ERF_HPP diff --git a/docker/rocm/migraphx/include/migraphx/match/gelu_tanh.hpp b/docker/rocm/migraphx/include/migraphx/match/gelu_tanh.hpp deleted file mode 100644 index 2e462e568..000000000 --- a/docker/rocm/migraphx/include/migraphx/match/gelu_tanh.hpp +++ /dev/null @@ -1,95 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MATCH_GELU_TANH_HPP -#define MIGRAPHX_GUARD_MATCH_GELU_TANH_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace match { - -namespace detail { -template -struct gelu_tanh_matcher -{ - F f; - - /// x ^ 3 - auto pow_fn() const { return f("pow")(used_once(), arg(1)(has_value(3.0))); } - - auto tanh_fn() const - { - /// Gelu tanh approximation - /// tanh( sqrt(2/M_PI) * (x + 0.044715 * x ^ 3 ) - auto mul_const_pow1 = f("mul")(either_arg(0, 1)(has_value(0.044715), pow_fn())); - auto add_any_mul = f("add")(any_arg(0, 1)(mul_const_pow1)); - auto mul_sqrt2rpi_add = f("mul")(either_arg(0, 1)(has_value(sqrt(M_2_PI)), add_any_mul)); - - /// FastGelu tanh approximation - /// tanh( 0.797885 * x + 0.035677 * x ^ 3 ) - auto mul_const_pow2 = f("mul")(either_arg(0, 1)(has_value(0.035677), pow_fn())); - auto mul_const_x = f("mul")(any_arg(0, 1)(has_value(0.797885))); - auto add_mul_x_mul_pow = f("add")(either_arg(0, 1)(mul_const_pow2, mul_const_x)); - return f("tanh")(used_once(), arg(0)(any_of(add_mul_x_mul_pow, mul_sqrt2rpi_add))); - } - - /// x * (0.5? + 0.5 * tanh( sqrt(2/M_PI) * (x? + 0.044715 * x? ^ 3) ) ) or - /// x * (0.5? + 0.5 * tanh( 0.797885 * x? + 0.035677 * x? ^ 3 ) ) - /// ? question mark means it doesn't explicitly match that item (anything will work) - auto matcher_v0() const - { - auto mul_half_tanh = f("mul")(either_arg(0, 1)(has_value(0.5), tanh_fn())); - auto add_any_mul = f("add")(any_arg(0, 1)(mul_half_tanh)); - return f("mul")(either_arg(0, 1)(any().bind("x"), add_any_mul)); - } - - /// x * 0.5 * (1.0 + tanh( sqrt(2/M_PI) * (x + 0.044715 * x ^ 3) ) ) or - /// x * 0.5 * (1.0 + tanh( 0.797885 * x + 0.035677 * x ^ 3 ) ) ) - auto matcher_v1() const - { - auto add_one_tanh = f("add")(used_once(), either_arg(0, 1)(has_value(1.0), tanh_fn())); - auto mul_half_x = f("mul")(used_once(), either_arg(0, 1)(has_value(0.5), any().bind("x"))); - return f("mul")(either_arg(0, 1)(mul_half_x, add_one_tanh)); - } -}; -} // namespace detail - -template -auto gelu_tanh(F f) -{ - auto gtm = detail::gelu_tanh_matcher{f}; - return any_of(gtm.matcher_v0(), gtm.matcher_v1()); -} - -inline auto gelu_tanh() -{ - return gelu_tanh([](auto x) { return name(x); }); -} - -} // namespace match -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MATCH_GELU_TANH_HPP diff --git a/docker/rocm/migraphx/include/migraphx/match/layernorm.hpp b/docker/rocm/migraphx/include/migraphx/match/layernorm.hpp deleted file mode 100644 index 83f809d1d..000000000 --- a/docker/rocm/migraphx/include/migraphx/match/layernorm.hpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_MATCH_LAYERNORM_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_MATCH_LAYERNORM_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace match { - -namespace detail { -template -struct layernorm_matcher -{ - F f; - - auto last_axis() const - { - return make_basic_pred_matcher([](instruction_ref ins) { - auto v = ins->get_operator().to_value(); - if(not v.contains("axes")) - return false; - auto axes = v["axes"].to_vector(); - if(axes.size() != 1) - return false; - return axes.front() == ins->inputs().front()->get_shape().lens().size() - 1; - }); - } - - auto reduce_mean() const { return f("reduce_mean")(last_axis()); } - - auto x_minus_mean() const - { - return f("sub")(arg(0)(any().bind("x")), arg(1)(skip_broadcasts(reduce_mean()))); - } - - auto variance() const - { - return reduce_mean()(arg(0)(any_of( - f("pow")(arg(0)(x_minus_mean()), arg(1)(has_value(2.0f))), - f("mul")(arg(0)(x_minus_mean()), arg(1)(x_minus_mean())), - f("sqdiff")(either_arg(0, 1)(any().bind("x"), skip_broadcasts(reduce_mean())))))); - } - - auto sqrt_add_eps(const std::string& name) const - { - auto add_eps = f("add")(either_arg(0, 1)(variance(), is_constant().bind("eps"))); - return skip_broadcasts(f(name)(arg(0)(any_of(add_eps, variance())))); - } - - auto layernorm_onnx() const - { - auto div_sqrt = f("div")(arg(0)(x_minus_mean()), arg(1)(sqrt_add_eps("sqrt"))); - auto mul_rsqrt = f("mul")(either_arg(0, 1)(x_minus_mean(), sqrt_add_eps("rsqrt"))); - return any(any_of(div_sqrt, mul_rsqrt)); - } - - auto matcher() const { return layernorm_onnx(); } -}; -} // namespace detail - -template -auto layernorm(F f) -{ - return detail::layernorm_matcher{f}.matcher(); -} - -inline auto layernorm() -{ - return layernorm([](auto x) { return name(x); }); -} - -} // namespace match -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/match/softmax.hpp b/docker/rocm/migraphx/include/migraphx/match/softmax.hpp deleted file mode 100644 index 40d2642ee..000000000 --- a/docker/rocm/migraphx/include/migraphx/match/softmax.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MATCH_SOFTMAX_HPP -#define MIGRAPHX_GUARD_MATCH_SOFTMAX_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace match { - -namespace detail { -template -struct softmax_matcher -{ - F f; - - auto exp_x_minus_max() const - { - auto x_minus_max = - f("sub")(arg(0)(any().bind("x")), arg(1)(skip_broadcasts(f("reduce_max")))); - return f("exp")(arg(0)(x_minus_max)); - } - - auto softmax_base_ops() const - { - auto sum_exp_x_minus_max = f("reduce_sum")(arg(0)(exp_x_minus_max())); - return f("div")(arg(0)(exp_x_minus_max()), arg(1)(skip_broadcasts(sum_exp_x_minus_max))); - } - - auto matcher() const { return softmax_base_ops(); } -}; -} // namespace detail - -template -auto softmax(F f) -{ - return detail::softmax_matcher{f}.matcher(); -} - -inline auto softmax() -{ - return softmax([](auto x) { return name(x); }); -} - -} // namespace match -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/matcher.hpp b/docker/rocm/migraphx/include/migraphx/matcher.hpp deleted file mode 100644 index 0babe316d..000000000 --- a/docker/rocm/migraphx/include/migraphx/matcher.hpp +++ /dev/null @@ -1,1009 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MATCHER_HPP -#define MIGRAPHX_GUARD_RTGLIB_MATCHER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef MIGRAPHX_USE_TYPE_ERASED_MATCHERS -#define MIGRAPHX_USE_TYPE_ERASED_MATCHERS 0 -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace match { - -struct matcher_context -{ - matcher_context(module& m) : mod(&m) {} - std::unordered_map instructions; - - template - bool matched(M m, instruction_ref ins) - { - return has_value(m.match(*this, ins)); - } - - template - bool matched(M m, optional ins) - { - if(ins) - return has_value(m.match(*this, *ins)); - return false; - } - - template - auto lazy_match(M m, I ins) - { - return [=] { return this->matched(m, ins); }; - } - - bool has_instruction(instruction_ref ins) const - { - if(mod == nullptr) - return true; - return mod->has_instruction(ins); - } - bool has_instruction(optional ins) const - { - if(ins) - return this->has_instruction(*ins); - return false; - } - - bool is_last(instruction_ref ins) const - { - assert(mod->begin() != mod->end()); - assert(this->has_instruction(ins)); - return ins == std::prev(mod->end()); - } - - void debug_print(instruction_ref ins) const { mod->debug_print(ins); } - - private: - module* mod = nullptr; -}; - -/// Convert a predicate function into a matcher -template -struct predicate_matcher -{ - P p; - - optional match(const matcher_context&, instruction_ref ins) const - { - if(p(ins)) - return optional{ins}; - return nullopt; - } -}; - -/// Convert a predicate function into a matcher -template -predicate_matcher

make_predicate_matcher(P p) -{ - return {p}; -} - -/// Convert a function into a matcher -template -struct function_matcher -{ - F f; - - auto match(matcher_context& ctx, instruction_ref ins) const { return f(ctx, ins); } -}; - -/// Convert a function into a matcher -template -function_matcher make_function_matcher(F f) -{ - return {f}; -} - -/// Converts a matcher to bind the instruction to name -template -auto bind_match(M m, std::string name) -{ - return make_function_matcher( - [=, m_name = std::move(name)](matcher_context& ctx, - instruction_ref ins) -> optional { - auto result = m.match(ctx, ins); - if(result) - { - if(not ctx.has_instruction(ins)) - return nullopt; - ctx.instructions[m_name] = ins; - } - return result; - }); -} - -/// Convert a matcher to a bindable matcher -template -struct bindable_matcher -{ - M m; - - auto bind(std::string name) const { return bind_match(m, std::move(name)); } - - auto match(matcher_context& ctx, instruction_ref ins) const { return m.match(ctx, ins); } -}; - -/// Create a bindable matcher -template -bindable_matcher make_bindable_matcher(M m) -{ - return {m}; -} - -/// Create a bindable matcher from a function -template -bindable_matcher> make_bf_matcher(F f) -{ - return {{f}}; -} - -/// Create a bindable matcher from a predicate function -template -bindable_matcher> make_bp_matcher(F f) -{ - return {{f}}; -} - -using bool_list = std::initializer_list; - -struct id_matcher -{ - auto match(matcher_context&, instruction_ref ins) const - { - return optional{ins}; - } -}; - -// Forward declare class and constructors -template -struct basic_matcher; - -struct any_matcher; - -template -struct type_erased_matcher -{ -#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS - using type = any_matcher; -#else - using type = basic_matcher; -#endif -}; - -template -typename type_erased_matcher::type make_basic_matcher(M m); - -template -auto make_basic_fun_matcher(F f); - -template -auto make_basic_pred_matcher(P p); - -/// The basic matcher provides the all_of composability of the matcher -template -struct basic_matcher -{ - M m; - - template - auto operator()(Ts... ms) const - { - // Copy m because we cant capture `this` by value - auto mm = m; - return make_basic_fun_matcher([=](matcher_context& ctx, - instruction_ref ins) -> optional { - auto result = mm.match(ctx, ins); - if(result) - { - bool matches = - fold([&](auto x, auto y) { return x and ctx.matched(y, result); })(true, ms...); - if(matches) - return result; - } - return nullopt; - }); - } - - auto bind(std::string name) const { return bind_match(m, std::move(name)); } - - auto match(matcher_context& ctx, instruction_ref ins) const { return m.match(ctx, ins); } -}; - -/// Create a typed-erased matcher -using any_matcher_base = basic_matcher< - function_matcher(matcher_context&, instruction_ref)>>>; -struct any_matcher : any_matcher_base -{ - template - any_matcher(M mm) : any_matcher_base({[=](auto& ctx, auto ins) { return mm.match(ctx, ins); }}) - { - } -}; - -/// Create a basic matcher from a matcher -template -typename type_erased_matcher::type make_basic_matcher(M m) -{ - return {m}; -} - -/// Create a basic matcher from a function -template -auto make_basic_fun_matcher(F f) -{ - return make_basic_matcher(make_function_matcher(f)); -} - -/// Create a basic matcher from a predicate function -template -auto make_basic_pred_matcher(P p) -{ - return make_basic_matcher(make_predicate_matcher(p)); -} - -/// This macro takes care of the boilerplate for defining a matcher -#define MIGRAPHX_BASIC_MATCHER(name, ...) \ - struct name##_m \ - { \ - optional match(__VA_ARGS__) const; \ - }; \ - const constexpr auto name = migraphx::match::basic_matcher{{}}; \ - inline optional name##_m::match(__VA_ARGS__) const - -/// This macro takes care of the boilerplate for defining a predicate matcher -#define MIGRAPHX_PRED_MATCHER(name, ...) \ - struct name##_m \ - { \ - bool operator()(__VA_ARGS__) const; \ - }; \ - const constexpr auto name = \ - migraphx::match::basic_matcher>{{}}; \ - inline bool name##_m::operator()(__VA_ARGS__) const - -struct matcher_result -{ - struct instruction_container - { - instruction_container() = default; - instruction_container(std::unordered_map x) - : ins_map(std::move(x)) - { - } - - instruction_ref operator[](const std::string& name) const - { - auto it = ins_map.find(name); - if(it == ins_map.end()) - MIGRAPHX_THROW("Accessing name that wasn't bound in matcher: " + name); - return it->second; - } - - auto find(const std::string& name) const { return ins_map.find(name); } - - auto begin() const { return ins_map.cbegin(); } - - auto end() const { return ins_map.cend(); } - - bool has_instructions_in(const module& mod) const - { - return std::all_of(ins_map.begin(), ins_map.end(), [&](auto&& p) { - return mod.has_instruction(p.second); - }); - } - - void debug_print() const - { - for(const auto& it : ins_map) - { - std::cout << it.first << ": \n"; - it.second->debug_print(); - } - } - - private: - std::unordered_map ins_map; - }; - - void debug_print() const - { - std::cout << "matcher_container: \n instructions:"; - instructions.debug_print(); - std::cout << " result: \n"; - result->debug_print(); - } - - instruction_container instructions; - instruction_ref result; -}; - -/// Match a single instruction -template -matcher_result match_instruction(module& mod, instruction_ref ins, M&& m) -{ - assert(ins != mod.end()); - assert(mod.has_instruction(ins)); - matcher_context ctx{mod}; - matcher_result result; - if(m.match(ctx, ins)) - { - result.result = ins; - result.instructions = ctx.instructions; - assert(result.instructions.has_instructions_in(mod)); - } - else - { - result.result = mod.end(); - } - return result; -} - -template -bool instruction_matches(module& mod, instruction_ref ins, M&& m) -{ - return match_instruction(mod, ins, std::forward(m)).result != mod.end(); -} - -/// Find first instance of a matching instruction in a module -template -match::matcher_result find_match(module& modl, M&& m) -{ - match::matcher_result result; - for(auto ins : iterator_for(modl)) - { - result = match::match_instruction(modl, ins, m); - if(result.result != modl.end()) - return result; - } - return result; -} - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_MATCHES) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_MATCHES_FOR) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_VALIDATE_MATCHES) - -/// Find matches for an instruction in the module for per section of matchers -template -void find_matches_for(source_location location, Mod& mod, instruction_ref ins, Ms&&... ms) -{ - const int trace = value_of(MIGRAPHX_TRACE_MATCHES{}); - const bool validate = enabled(MIGRAPHX_VALIDATE_MATCHES{}); - const auto trace_filter = string_value_of(MIGRAPHX_TRACE_MATCHES_FOR{}); - bool match = false; - each_args( - [&](auto&& m) { - const auto& matcher_name = get_type_name(m); - const bool trace_for = not trace_filter.empty() and - (contains(std::string{location.file_name()}, trace_filter) or - contains(std::string{location.function_name()}, trace_filter) or - contains(matcher_name, trace_filter)); - if(match) - return; - if(trace > 1 and trace_for) - std::cout << "Match: " << matcher_name << std::endl; - auto r = match_instruction(get_module(mod), ins, m.matcher()); - if(r.result == get_module(mod).end()) - return; - if(trace > 0 or trace_for) - { - std::cout << "Matched by " << matcher_name << std::endl; - get_module(mod).debug_print(ins); - } - // If its already invalid dont validate it again - bool invalidated = validate and get_module(mod).validate() != get_module(mod).end(); - m.apply(mod, r); - if(validate and not invalidated) - { - auto invalid = get_module(mod).validate(); - if(invalid != get_module(mod).end()) - { - std::cout << "Invalid program from match: " << matcher_name << std::endl; - std::cout << "Invalid instructions: " << std::endl; - get_module(mod).debug_print(invalid->inputs()); - get_module(mod).debug_print(invalid); - } - } - match = true; - }, - ms...); -} - -/// Find matches in a module -template -struct find_matches -{ - find_matches(Mod& mod, Ms&&... ms, source_location location = source_location::current()) - { - for(auto ins : iterator_for(get_module(mod))) - { - find_matches_for(location, mod, ins, ms...); - } - } -}; - -template -find_matches(Mod& mod, Ms&&... ms) -> find_matches; - -template -struct find_generic_match -{ - M m; - F f; - M matcher() const { return m; } - - void apply(module& mod, const matcher_result& mr) const { f(mod, mr); } -}; - -template -find_generic_match make_match_finder(M m, F f) -{ - return {m, f}; -} - -template -struct find_skip -{ - M m; - M matcher() const { return m; } - - void apply(module&, const matcher_result&) const {} -}; - -template -find_skip make_find_skip(M m) -{ - return {m}; -} - -struct lazy_and -{ - template - auto operator()(F f, G g) const - { - return [=] { return f() and g(); }; - } -}; - -struct lazy_or -{ - template - auto operator()(F f, G g) const - { - return [=] { return f() or g(); }; - } -}; - -template -struct match_fold_f -{ - template - static bool fold_matchers(matcher_context& ctx, instruction_ref ins, Ms... ms) - { - Op op; - auto matched = [&](auto m) { return [=, &ctx] { return ctx.matched(m, ins); }; }; - return fold(op)(always(Start), matched(ms)...)(); - } - - template - static bool fold_matchers_pack(matcher_context& ctx, instruction_ref ins, Pack p) - { - return p([&](auto... ms) { return match_fold_f::fold_matchers(ctx, ins, ms...); }); - } - - template - auto operator()(Ts... ms) const - { - return make_bf_matcher( - [=](matcher_context& ctx, instruction_ref ins) -> optional { - bool matches = match_fold_f::fold_matchers(ctx, ins, ms...); - if(matches == Matches) - return {ins}; - return nullopt; - }); - } - - template - auto operator[](Selector select) const - { - return [=](auto... ms) { - // Workaround ICE on gcc by packing matchers into an object - auto mpack = pack(ms...); - return make_bf_matcher( - [=](matcher_context& ctx, instruction_ref start) -> optional { - Op op; - bool matches = Start; - select(start, [&](auto ins) { - auto fm = [&] { return match_fold_f::fold_matchers_pack(ctx, ins, mpack); }; - matches = op(always(matches), fm)(); - }); - if(matches == Matches) - return {start}; - return nullopt; - }); - }; - } -}; - -const constexpr auto all_of = match_fold_f{}; -const constexpr auto any_of = match_fold_f{}; -const constexpr auto none_of = match_fold_f{}; - -template -auto skip_matches(Ms... ms) -{ - return make_find_skip(any_of(ms...)); -} - -inline auto inputs() -{ - return [](auto ins, auto f) { - for(auto&& x : ins->inputs()) - f(x); - }; -} - -inline auto outputs() -{ - return [](auto ins, auto f) { - for(auto&& x : ins->outputs()) - f(x); - }; -} - -inline auto trace(const std::string& s) -{ - return [=](auto m) { - return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref ins) { - std::cout << s << ": "; - ctx.debug_print(ins); - optional result = m.match(ctx, ins); - if(result.has_value()) - std::cout << "Found\n"; - else - std::cout << "Not Found\n"; - return result; - }); - }; -} - -inline auto trace_found(const std::string& s) -{ - return [=](auto m) { - return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref ins) { - optional result = m.match(ctx, ins); - if(result.has_value()) - { - std::cout << "Found: " << s << ": "; - ctx.debug_print(ins); - } - return result; - }); - }; -} - -inline auto trace_not_found(const std::string& s) -{ - return [=](auto m) { - return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref ins) { - optional result = m.match(ctx, ins); - if(not result.has_value()) - { - std::cout << "Not Found: " << s << ": "; - ctx.debug_print(ins); - } - return result; - }); - }; -} - -MIGRAPHX_PRED_MATCHER(any, instruction_ref) { return true; } -MIGRAPHX_PRED_MATCHER(none, instruction_ref) { return false; } -MIGRAPHX_PRED_MATCHER(standard_shape, instruction_ref ins) { return ins->get_shape().standard(); } -MIGRAPHX_PRED_MATCHER(not_standard_shape, instruction_ref ins) -{ - return not ins->get_shape().standard(); -} -MIGRAPHX_PRED_MATCHER(dynamic_shape, instruction_ref ins) { return ins->get_shape().dynamic(); } -MIGRAPHX_PRED_MATCHER(static_shape, instruction_ref ins) { return not ins->get_shape().dynamic(); } -MIGRAPHX_PRED_MATCHER(broadcast_shape, instruction_ref ins) -{ - return ins->get_shape().broadcasted(); -} - -MIGRAPHX_PRED_MATCHER(scalar_shape, instruction_ref ins) { return ins->get_shape().scalar(); } - -MIGRAPHX_PRED_MATCHER(transpose_shape, instruction_ref ins) -{ - return ins->get_shape().transposed(); -} - -MIGRAPHX_PRED_MATCHER(same_input_shapes, instruction_ref ins) -{ - if(ins->inputs().empty()) - return false; - auto s = ins->inputs().front()->get_shape(); - return std::all_of( - ins->inputs().begin(), ins->inputs().end(), [&](auto x) { return x->get_shape() == s; }); -} - -MIGRAPHX_PRED_MATCHER(same_inputs, instruction_ref ins) -{ - if(ins->inputs().empty()) - return false; - auto input = ins->inputs().front(); - return std::all_of( - ins->inputs().begin(), ins->inputs().end(), [&](auto x) { return x == input; }); -} - -MIGRAPHX_PRED_MATCHER(has_same_value, instruction_ref ins) -{ - if(ins->name() != "@literal") - return false; - bool all_same = false; - ins->get_literal().visit([&](auto s) { - all_same = std::all_of(s.begin() + 1, s.end(), [&](const auto& scale) { - return float_equal(scale, s.front()); - }); - }); - return all_same; -} - -MIGRAPHX_BASIC_MATCHER(output, const matcher_context&, instruction_ref ins) -{ - if(ins->outputs().size() == 1) - return {ins->outputs().front()}; - return nullopt; -} - -MIGRAPHX_BASIC_MATCHER(used_once, const matcher_context& ctx, instruction_ref ins) -{ - if(ins->outputs().size() == 1) - return {ins}; - if(ins->outputs().empty() and ctx.is_last(ins)) - return {ins}; - return nullopt; -} - -MIGRAPHX_PRED_MATCHER(is_constant, instruction_ref ins) { return ins->can_eval(); } - -MIGRAPHX_BASIC_MATCHER(is_unused, const matcher_context& ctx, instruction_ref ins) -{ - if(ins->outputs().empty() and not ctx.is_last(ins)) - return {ins}; - return nullopt; -} - -MIGRAPHX_PRED_MATCHER(broadcast, instruction_ref ins) -{ - return contains({"broadcast", "multibroadcast"}, ins->name()); -} - -template -auto skip(Ms... ms) -{ - static_assert(((not std::is_convertible{}) and ...), - "Use a matcher not a string for skip."); - auto m = any_of(ms...); - return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref start) { - return fix>( - [&](auto self, auto ins) -> optional { - if(ins->inputs().size() == 1 and ctx.matched(m, ins)) - { - auto next = ins->inputs().front(); - return self(next); - } - return ins; - })(start); - }); -} - -template -auto skip_output(Ms... ms) -{ - auto m = any_of(ms...); - return make_basic_fun_matcher([=](matcher_context& ctx, instruction_ref start) { - return fix>( - [&](auto self, auto ins) -> optional { - if(ins->outputs().size() == 1) - { - auto next = ins->outputs().front(); - if(ctx.matched(m, next)) - { - auto skipped_next = self(next); - if(skipped_next) - return skipped_next; - } - return next; - } - return nullopt; - })(start); - }); -} - -inline auto var(std::string s) -{ - return make_basic_fun_matcher( - [=, m_s = std::move(s)](const matcher_context& ctx, - instruction_ref) -> optional { - auto it = ctx.instructions.find(m_s); - if(it == ctx.instructions.end()) - return nullopt; - return it->second; - }); -} - -inline auto name(std::string s) -{ - return make_basic_pred_matcher( - [=, m_s = std::move(s)](instruction_ref ins) { return ins->name() == m_s; }); -} - -inline auto name_contains(const std::string& name) -{ - return make_basic_pred_matcher( - [=](instruction_ref ins) { return contains(ins->get_operator().name(), name); }); -} - -inline auto name(std::unordered_set names) -{ - return make_basic_pred_matcher([=, m_names = std::move(names)](instruction_ref ins) { - return m_names.count(ins->name()) > 0; - }); -} - -template -inline auto name(std::string s, Ts... xs) // NOLINT -{ - return name(std::unordered_set{std::move(s), std::move(xs)...}); -} - -inline auto nargs(std::size_t n) -{ - return make_basic_pred_matcher([=](instruction_ref ins) { return ins->inputs().size() == n; }); -} - -inline auto arg(std::size_t i) -{ - return make_basic_fun_matcher( - [=](const matcher_context&, instruction_ref ins) -> optional { - if(i < ins->inputs().size()) - return ins->inputs()[i]; - return nullopt; - }); -} - -// Workaround for bugs in clang -template -struct args_impl_ints -{ -}; - -template -auto args_impl(args_impl_ints, Ms... ms) -{ - return match::all_of(nargs(sizeof...(Ns)), arg(Ns)(ms)...); -} - -template -auto args(Ms... ms) -{ - return sequence_c([=](auto... is) { - // It needs to be written as `decltype(is)::value` for gcc 5 - return args_impl(args_impl_ints{}, ms...); - }); -} - -inline auto either_arg(std::size_t i, std::size_t j) -{ - return [=](auto m1, auto m2) { - return match::any_of(match::all_of(arg(i)(m1), arg(j)(m2)), - match::all_of(arg(j)(m1), arg(i)(m2))); - }; -} - -inline auto any_arg(std::size_t i, std::size_t j) -{ - return [=](auto m) { return match::any_of(arg(i)(m), arg(j)(m)); }; -} - -template -std::size_t tree_leafs_impl(matcher_context& ctx, - std::array& leafs, - M m, - instruction_ref ins) -{ - std::size_t idx = 0; - fix([&](auto self, auto i) { - if(idx == leafs.size()) - return; - if(ctx.matched(m, i) and i->inputs().size() >= 2) - { - self(i->inputs()[0]); - self(i->inputs()[1]); - return; - } - leafs[idx] = i; - idx++; - })(ins); - return idx; -} - -template -auto tree(M main_op, Ms... ms) -{ - return make_basic_fun_matcher( - [=](matcher_context& ctx, instruction_ref ins) -> optional { - // Flatten leaf nodes - std::array leafs; - std::size_t idx = tree_leafs_impl(ctx, leafs, main_op, ins); - if(idx != leafs.size()) - return nullopt; - // Use explicit captures to workaround ICE on gcc - // Capture by value to workaround compile error on gcc 9 - bool found = sequence_c([ms..., &ctx, &leafs](auto... is) { - return fold(lazy_and{})(ctx.lazy_match(ms, leafs[is])...)(); - }); - if(not found) - return nullopt; - return ins; - }); -} - -template -auto unordered_tree(M main_op, Ms... ms) -{ - return make_basic_fun_matcher( - [=](matcher_context& ctx, instruction_ref ins) -> optional { - // Flatten leaf nodes - std::array leafs; - std::size_t idx = tree_leafs_impl(ctx, leafs, main_op, ins); - if(idx != leafs.size()) - return nullopt; - // Use explicit captures to workaround ICE on gcc - bool found = sequence_c([ms..., &ctx, &leafs](auto... is) { - return by(fold(lazy_and{}), [is..., &ctx, &leafs](auto m) { - return fold(lazy_or{})(ctx.lazy_match(m, leafs[is])...); - })(ms...)(); - }); - if(not found) - return nullopt; - return ins; - }); -} - -template -auto same_shape(M m) -{ - return make_basic_fun_matcher( - [=](matcher_context& ctx, instruction_ref ins) -> optional { - auto i = m.match(ctx, ins); - if(i and (*i)->get_shape() == ins->get_shape()) - return ins; - return nullopt; - }); -} - -template -auto same_shape(Ms... ms) -{ - return all_of(same_shape(ms)...); -} - -template -auto skip_broadcasts(Ms... ms) -{ - return skip(name("broadcast", "multibroadcast", "contiguous"))(ms...); -} - -template -auto skip_broadcasts_converts(Ms... ms) -{ - return skip(name("broadcast", "multibroadcast", "contiguous", "convert"))(ms...); -} - -template -inline auto literal_value_checker(F f) -{ - return skip_broadcasts_converts(make_basic_pred_matcher([=](instruction_ref ins) { - if(ins->name() != "@literal") - return false; - auto l = ins->get_literal(); - if(l.empty()) - return false; - return f(l); - })); -} - -/** - * Uses integer multiples of the corresponding floating point epsilon and - * compares with abs(y - x) < eps * (atol_mult + rtol_mult * abs(x)). - * atol_mult controls the absolute tolerance. - * rtol_mult controls the relative tolerance. - * Uses no tolerance for integral types. - */ -template -inline auto has_value(T x, std::size_t atol_mult = 10, std::size_t rtol_mult = 10) -{ - return literal_value_checker([=](migraphx::literal l) { - bool b = false; - l.visit([&](auto v) { - // cast to the literal's data type before comparing - using type = typename decltype(v)::value_type; - auto tolerance = atol_mult + rtol_mult * std::fabs(x); - if(migraphx::float_equal(tolerance, 0) or std::is_integral{}) - { - if(std::all_of(v.begin(), v.end(), [&](auto val) { - return migraphx::float_equal(val, static_cast(x)); - })) - b = true; - } - else - { - auto eps = std::numeric_limits::epsilon(); - if(std::all_of(v.begin(), v.end(), [&](auto val) { - return std::fabs(val - static_cast(x)) < (eps * tolerance); - })) - b = true; - } - }); - return b; - }); -} - -inline auto has_attribute(const std::string& name) -{ - return make_basic_pred_matcher( - [=](instruction_ref ins) { return ins->get_operator().attributes().contains(name); }); -} - -template -auto pointwise(Ms... ms) -{ - return match::has_attribute("pointwise")(ms...); -} - -} // namespace match -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/memory_coloring.hpp b/docker/rocm/migraphx/include/migraphx/memory_coloring.hpp deleted file mode 100644 index 27f07367a..000000000 --- a/docker/rocm/migraphx/include/migraphx/memory_coloring.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MEMORY_COLORING_HPP -#define MIGRAPHX_GUARD_RTGLIB_MEMORY_COLORING_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct module; - -/** - * Remove multiple memory allocations using graph coloring to find memory allocations that can be - * reused. - */ -struct MIGRAPHX_EXPORT memory_coloring -{ - std::string allocation_op{}; - bool verify = false; - std::string name() const { return "memory_coloring"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/module.hpp b/docker/rocm/migraphx/include/migraphx/module.hpp deleted file mode 100644 index 68734c82b..000000000 --- a/docker/rocm/migraphx/include/migraphx/module.hpp +++ /dev/null @@ -1,366 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_MODULE_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_MODULE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT -const operation& get_operation(instruction_ref ins); - -struct module_impl; - -using parameter_map = std::unordered_map; -using ins_dep_map = std::unordered_map>; - -struct module_with_inputs; - -/** - * @brief Stores the instruction stream - */ -struct MIGRAPHX_EXPORT module -{ - using inserter = std::function& inputs, - const std::vector& mod_args)>; - module(const std::string& name = ""); - - // move constructor - module(module&&) noexcept; - - // copy constructor - module(const module&); - - // copy assignment operator - module& operator=(module); - - ~module() noexcept; - - std::string name() const; - - bool bypass() const; - void set_bypass(bool b = true); - - template {}...)> - instruction_ref add_instruction(operation op, Ts... args) - { - return add_instruction(op, {args...}); - } - - instruction_ref add_instruction(const operation& op, std::vector args); - - instruction_ref add_instruction(const operation& op, - std::vector args, - std::vector module_args); - - template {}...)> - instruction_ref insert_instruction(instruction_ref ins, operation op, Ts... args) - { - return insert_instruction(ins, op, {args...}); - } - instruction_ref - insert_instruction(instruction_ref ins, const operation& op, std::vector args); - - instruction_ref insert_instruction(instruction_ref ins, - const operation& op, - std::vector args, - std::vector module_args); - - template {}...)> - instruction_ref replace_instruction(instruction_ref ins, operation op, Ts... args) - { - return replace_instruction(ins, op, {args...}); - } - instruction_ref replace_instruction(instruction_ref ins, - const operation& op, - std::vector args) MIGRAPHX_TIDY_CONST; - - instruction_ref replace_instruction(instruction_ref ins, - const operation& op, - std::vector args, - std::vector module_args) MIGRAPHX_TIDY_CONST; - - instruction_ref replace_instruction(instruction_ref ins, instruction_ref rep); - - instruction_ref remove_instruction(instruction_ref ins); - instruction_ref remove_instructions(instruction_ref first, instruction_ref last); - - instruction_ref move_instruction(instruction_ref src, instruction_ref dst); - instruction_ref move_instructions(instruction_ref src, instruction_ref dst); - - std::vector - add_instructions(const std::vector& instructions, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - std::vector - add_instructions(const_module_ref m, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - std::vector - add_instructions(instruction_ref start, - instruction_ref last, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - std::vector - insert_instructions(instruction_ref ins, - const std::vector& instructions, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - std::vector - insert_instructions(instruction_ref ins, - const_module_ref m, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - std::vector - insert_instructions(instruction_ref ins, - instruction_ref start, - instruction_ref last, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - template - instruction_ref add_literal(Ts&&... xs) - { - return add_literal(literal{std::forward(xs)...}); - } - - instruction_ref add_literal(literal l); - - instruction_ref add_outline(const shape& s); - - instruction_ref add_parameter(std::string name, shape s); - - instruction_ref add_return(std::vector args); - - instruction_ref replace_return(std::vector args); - - instruction_ref insert_literal(instruction_ref ins, literal l); - - instruction_ref insert_parameter(instruction_ref ins, std::string name, shape s); - - std::vector get_parameter_names() const; - - shape get_parameter_shape(std::string name) const; - - instruction_ref get_parameter(std::string name) const; - - std::vector get_parameters() const; - - void rename_parameter(instruction_ref ins, const std::string& name); - - std::unordered_map get_parameter_shapes() const; - - bool has_instruction(instruction_ref ins) const; - - std::vector get_returns() const; - - std::size_t size() const; - instruction_ref begin() const; - instruction_ref end() const; - - struct compute_shapes_options - { - std::string name = "compute_shapes"; - bool strict_type = false; - bool strict_lens = false; - std::vector scalar_const_out_lens = {}; - }; - - /// Compute a new ouput shape by replacing each parameter with input - /// shapes passed in. - std::vector compute_shapes(const std::vector& inputs, - compute_shapes_options options) const; - std::vector compute_shapes(const std::vector& inputs) const; - - std::vector get_output_shapes() const; - - instruction_ref validate() const; - instruction_ref find_dangling_reference() const; - - void finalize(std::vector& contexts); - - /// Create a mapping from the input instruction to the corresponding - /// parameter instruction. Use the `reverse` flag to reverse the lookup - /// to be from parameter instruction to input instread. - std::unordered_map - get_ins_param_map(const std::vector& inputs, bool reverse = false) const; - - /// Given a mapping from submodule instructions to parent module instructions - /// construct a vector of inputs with parent module instructions in the - /// correct order - std::vector - get_inputs(const std::unordered_map& map_ins) const; - - using with_inputs = module_with_inputs; - - /// This will split the module into two parts at the instruction splits. - /// Each split instruction becomes an input parameter in the second - /// module. As such the inputs instructions to the second module will use - /// the split instructions as input placeholders that can be replaced - /// later. - std::array split(const std::vector& args, - const std::vector& splits) const; - - /// This will split the module in 3 parts using different split - /// instruction for each additional module. - std::array split(const std::vector& args, - const std::vector& splits1, - const std::vector& splits2) const; - - // Insert params to module based on given input instructions and add - // mappings from inputs to corresponding params in instructions map - void add_params(const std::vector& inputs, - std::unordered_map* map_ins = nullptr); - - // Fuse the instruction into the module by inserting the instructions and - // parameters for any missing inputs. - std::vector - fuse(const std::vector& inss, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - // Fuse another module into this module by inserting the instructions and - // parameters from the module - std::vector - fuse(const module& m, - const std::vector& inputs, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - /* - Insert instructions from module `m` to this module at position `ins` - */ - std::vector - insert_inline(instruction_ref ins, - const module& m, - const std::vector& inputs, - std::unordered_map* map_ins = nullptr, - inserter insert = nullptr); - - void debug_print() const; - void debug_print(instruction_ref ins) const; - void debug_print(instruction_ref ins, - std::unordered_map& names) const; - void debug_print(const std::vector& inss) const; - - std::unordered_map print( - const std::function&)>& print_func, - std::unordered_map names) const; - void print(const std::function&)>& - print_func) const; - - void print_graph(std::ostream& os, bool brief = false) const; - - void print_py(std::ostream& os) const; - std::unordered_map - print_py(std::ostream& os, - const std::string& mname, - std::unordered_map names) const; - - void print_cpp(std::ostream& os) const; - std::unordered_map - print_cpp(std::ostream& os, - const std::string& mname, - std::unordered_map names) const; - - void annotate(std::ostream& os, std::function a) const; - - std::vector get_sub_modules(bool shallow = false) const; - /* sorts the module in topological order aka reverse-post order (RPO) DFS order - it takes last instruction or @return as the root and walks back the graph and moves inputs - of the each instruction such that it appears before the instruction itself. - */ - module& sort(); - /* Any instruction "X" can have module arguments and those modules inside them can use any other - * instruction "Y" from predecessor modules of the instruction "X". Such instruction "Y" inside - * module args are not listed as input instructions to "X". But those instructions "Y" must be - * evaluted before the instruction "X" can. Therefore such "Y" instructions are considered - * implicit dependency to "X". - */ - ins_dep_map calc_implicit_deps() const; - - void repeat_while_changes(std::size_t n, const std::function& f); - - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const module& m); - MIGRAPHX_EXPORT friend bool operator==(const module& x, const module& y); - friend bool operator!=(const module& x, const module& y) { return not(x == y); } - - friend struct program; - - private: - void set_name(const std::string& name); - void assign(const module& m); - void calc_implicit_deps(const module& smod, - const module& pmod, - instruction_ref ins, - ins_dep_map& deps) const; - - std::unique_ptr impl; -}; - -struct MIGRAPHX_EXPORT module_with_inputs -{ - module mod; - std::vector inputs; - /// Replace the instruction in the inputs with rep - void replace(instruction_ref ins, instruction_ref rep); - /// Replace the input instructions using the map_ins to lookup the replacement - void replace(const std::unordered_map& map_ins); - - /// Replace the input instructions of the keys with the instructions - /// passed as values. Both vectors should be in the same order. - void replace(const std::vector& keys, - const std::vector& values); -}; - -inline module& get_module(module& m) { return m; } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/module_ref.hpp b/docker/rocm/migraphx/include/migraphx/module_ref.hpp deleted file mode 100644 index 807a39b72..000000000 --- a/docker/rocm/migraphx/include/migraphx/module_ref.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MODULE_REF_HPP -#define MIGRAPHX_GUARD_MODULE_REF_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -using module_ref = module*; -using const_module_ref = const module*; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/msgpack.hpp b/docker/rocm/migraphx/include/migraphx/msgpack.hpp deleted file mode 100644 index 83efd6bed..000000000 --- a/docker/rocm/migraphx/include/migraphx/msgpack.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MSGPACK_HPP -#define MIGRAPHX_GUARD_RTGLIB_MSGPACK_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT void to_msgpack(const value& v, - std::function writer); -MIGRAPHX_EXPORT std::vector to_msgpack(const value& v); -MIGRAPHX_EXPORT value from_msgpack(const std::vector& buffer); -MIGRAPHX_EXPORT value from_msgpack(const char* buffer, std::size_t size); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/netron_output.hpp b/docker/rocm/migraphx/include/migraphx/netron_output.hpp deleted file mode 100644 index fb355a2d9..000000000 --- a/docker/rocm/migraphx/include/migraphx/netron_output.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_NETRON_OUTPUT_HPP -#define MIGRAPHX_GUARD_RTGLIB_NETRON_OUTPUT_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT std::string make_netron_output(const program& prog); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/normalize_attributes.hpp b/docker/rocm/migraphx/include/migraphx/normalize_attributes.hpp deleted file mode 100644 index 61887af3f..000000000 --- a/docker/rocm/migraphx/include/migraphx/normalize_attributes.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_NORMALIZE_ATTRIBUTES_HPP -#define MIGRAPHX_GUARD_RTGLIB_NORMALIZE_ATTRIBUTES_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct operation; - -template -struct select_dependent_type -{ - using type = T; -}; -template -using dependent_type = typename select_dependent_type::type; - -/** - * Used to normalize variable input axes at model runtime. - * Example: the axes inputs of the slice operator. - * - * \param axes the axes to normalize - * \param input_shape shape of the input tensor - * \param attr_val the normalize_axes attributes from the operator - * \param prefix error message prefix - */ -MIGRAPHX_EXPORT -std::vector normalize_axes(const std::vector& axes, - const shape& input_shape, - const value& attr_val, - const std::string& prefix = ""); - -/** - * Used to normalize variable input axes at model runtime. - * Example: the starts and ends inputs of the slice operator. - * - * \param indices the indices to normalize - * \param axes which axes the indices apply over - * \param input_shape shape of the input tensor - * \param attr_val the normalize_axes attributes from the operator - * \param prefix error message prefix - */ -MIGRAPHX_EXPORT -std::vector normalize_indices(const std::vector& indices, - const std::vector& axes, - const shape& input_shape, - const value& attr_val, - const std::string& prefix = ""); - -MIGRAPHX_EXPORT -bool normalize_attributes(operation& op, const shape& input_shape); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/normalize_ops.hpp b/docker/rocm/migraphx/include/migraphx/normalize_ops.hpp deleted file mode 100644 index fced7a8e4..000000000 --- a/docker/rocm/migraphx/include/migraphx/normalize_ops.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_NORMALIZE_OPS_HPP -#define MIGRAPHX_GUARD_RTGLIB_NORMALIZE_OPS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Process negative axis attributes of ops - */ - -struct MIGRAPHX_EXPORT normalize_ops -{ - std::string name() const { return "normalize_ops"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/onnx.hpp b/docker/rocm/migraphx/include/migraphx/onnx.hpp deleted file mode 100644 index b9bf42a4f..000000000 --- a/docker/rocm/migraphx/include/migraphx/onnx.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_ONNX_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_ONNX_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// struct to pass in onnx options to parser -struct onnx_options -{ - /// Old way to set default fixed dimension size - std::size_t default_dim_value = 0; - /// Default dynamic dimension size (if both default_dim_value and default_dyn_dim_value set - /// parser throws) - shape::dynamic_dimension default_dyn_dim_value = {1, 1}; - /// Explicitly specify the dims of an input - std::unordered_map> map_input_dims = {}; - /// Explicitly specify a symbolic named parameter dimension - std::unordered_map dim_params = {}; - /// Explicitly specify dynamic dims of an input (if both map_input_dims and map_dyn_input_dims - /// set parser throws) - std::unordered_map> map_dyn_input_dims = {}; - /// Continue parsing onnx file if an unknown operator is found - bool skip_unknown_operators = false; - /// Print program if an error occurs - bool print_program_on_error = false; - /// Max iter num for the loop operator if trip count is not set - int64_t max_loop_iterations = 10; - /// Max iter limit for the loop operator. - /// Since loop will become a tensor of max iter size a huge number can cause overflow during - /// shape computations. - int64_t limit_max_iterations = std::numeric_limits::max(); - /// Use dynamic output for operators when available - bool use_dyn_output = false; - /// Path to use for the external data if it is stored at different location compared to onnx - /// file - std::string external_data_path = ""; -}; - -/// Create a program from an onnx file -MIGRAPHX_ONNX_EXPORT program parse_onnx(const std::string& name, - const onnx_options& = onnx_options{}); - -/// Create a program from an onnx buffer -MIGRAPHX_ONNX_EXPORT program parse_onnx_buffer(const std::string& buffer, - const onnx_options& options); - -/// Create a program from an onnx buffer -MIGRAPHX_ONNX_EXPORT program parse_onnx_buffer(const void* data, - std::size_t size, - const onnx_options& options); - -MIGRAPHX_ONNX_EXPORT std::vector get_onnx_operators(); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/abs.hpp b/docker/rocm/migraphx/include/migraphx/op/abs.hpp deleted file mode 100644 index 9f766520b..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/abs.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ABS_HPP -#define MIGRAPHX_GUARD_OPERATORS_ABS_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct abs : unary -{ - auto apply() const - { - return [](auto x) { return std::abs(make_signed(x)); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/acos.hpp b/docker/rocm/migraphx/include/migraphx/op/acos.hpp deleted file mode 100644 index 99f37fd85..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/acos.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ACOS_HPP -#define MIGRAPHX_GUARD_OPERATORS_ACOS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct acos : unary -{ - auto apply() const - { - return [](auto x) { return std::acos(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/acosh.hpp b/docker/rocm/migraphx/include/migraphx/op/acosh.hpp deleted file mode 100644 index b467a12df..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/acosh.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ACOSH_HPP -#define MIGRAPHX_GUARD_OPERATORS_ACOSH_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct acosh : unary -{ - auto apply() const - { - return [](auto x) { return std::acosh(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/add.hpp b/docker/rocm/migraphx/include/migraphx/op/add.hpp deleted file mode 100644 index 861d7ef00..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/add.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ADD_HPP -#define MIGRAPHX_GUARD_OPERATORS_ADD_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct add : binary -{ - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = true; - return a; - } - std::string point_function() const { return "+"; } - auto apply() const - { - return [](auto x, auto y) { return x + y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/allocate.hpp b/docker/rocm/migraphx/include/migraphx/op/allocate.hpp deleted file mode 100644 index 5b619f701..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/allocate.hpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ALLOCATE_HPP -#define MIGRAPHX_GUARD_OPERATORS_ALLOCATE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Static allocate: - * No inputs: `allocate()` - * `this.s` attribute set to the static output shape of the buffer. - * `this.s` attribute can be set to a dynamic output shape; however this will allocate the maximum - * buffer size for that case - * - * Dynamic allocate: - * One input: `allocate(output_dims)` - * `output_dims` are the output buffer dimensions and has a static shape. - * Either `this.s` or `this.buf_type` (but not both) must be set to calculate the dynamic output - * shape at compute time. If `this.buf_type` is set, the compute_shape() of allocate at compile time - * will have dynamic_dimensions from {0, max_int} with rank = output_dims.ndim(). If `this.s` is set - * then the compute_shape() will output `this.s`; `this.s` should be a dynamic shape. - */ -struct allocate -{ - optional s; - // for dynamic allocate to set the buffer type - optional buf_type; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape"), f(self.buf_type, "buf_type")); - } - - std::string name() const { return "allocate"; } - - shape compute_shape(const std::vector& inputs) const - { - if(s.has_value()) - { - if(buf_type.has_value()) - { - MIGRAPHX_THROW("ALLOCATE: shape and buf_type attributes both set"); - } - if(inputs.size() == 1) - { - migraphx::check_shapes{inputs, *this, false}.only_dims(1); - } - else - { - migraphx::check_shapes{inputs, *this, false}.has(0); - } - return s.value(); - } - else - { - if(not buf_type.has_value()) - { - MIGRAPHX_THROW("ALLOCATE: shape and buf_type attributes both not set"); - } - migraphx::check_shapes{inputs, *this, false}.has(1).only_dims(1); - const auto& out_dims = inputs.at(0); - std::size_t max_val = std::numeric_limits::max(); - std::vector dyn_dims(out_dims.lens().at(0), - shape::dynamic_dimension{0, max_val}); - return {buf_type.value(), dyn_dims}; - } - } - argument compute(const shape& output_shape, const std::vector& args) const - { - if(args.empty()) - { - return argument{output_shape}; - } - else - { - std::vector output_dims(output_shape.ndim()); - args.at(0).visit([&](auto a) { output_dims.assign(a.begin(), a.end()); }); - if(s) - { - return argument{shape{s->type(), output_dims}}; - } - return argument{shape{buf_type.value(), output_dims}}; - } - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/argmax.hpp b/docker/rocm/migraphx/include/migraphx/op/argmax.hpp deleted file mode 100644 index c78dc4f42..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/argmax.hpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ARGMAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_ARGMAX_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct argmax -{ - int64_t axis = 0; - bool select_last_index = false; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.select_last_index, "select_last_index")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "argmax"; } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - const auto& s0 = inputs[0]; - if(s0.dynamic()) - { - auto dyn_dims = s0.dyn_dims(); - dyn_dims[axis] = {1, 1}; - return {shape::int64_type, dyn_dims}; - } - else - { - auto lens = s0.lens(); - lens[axis] = 1; - return {shape::int64_type, lens}; - } - } - - template - int64_t calc_argmax(T& input, std::vector& indices, size_t item_num) const - { - auto max_val = input(indices.begin(), indices.end()); - int64_t max_index = 0; - for(std::size_t i = 1; i < item_num; ++i) - { - indices[axis] = i; - auto cur_val = input(indices.begin(), indices.end()); - if(max_val < cur_val) - { - max_val = cur_val; - max_index = i; - } - else if(select_last_index and float_equal(max_val, cur_val)) - { - max_index = i; - } - } - return max_index; - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - auto batch_item_num = args.front().get_shape().lens()[axis]; - - result.visit([&](auto output) { - args[0].visit([&](auto input) { - par_for(dyn_out.computed_shape.elements(), [&](auto i) { - auto data_idx = dyn_out.computed_shape.multi(i); - output[i] = this->calc_argmax(input, data_idx, batch_item_num); - }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/argmin.hpp b/docker/rocm/migraphx/include/migraphx/op/argmin.hpp deleted file mode 100644 index e0f43e057..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/argmin.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ARGMIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_ARGMIN_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct argmin -{ - int64_t axis = 0; - bool select_last_index = false; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.select_last_index, "select_last_index")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "argmin"; } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1); - auto lens = inputs[0].lens(); - - lens[axis] = 1; - - return {shape::int64_type, lens}; - } - - template - int64_t calc_argmin(T& input, std::vector& indices, size_t item_num) const - { - auto min_val = input(indices.begin(), indices.end()); - int64_t min_index = 0; - for(std::size_t i = 1; i < item_num; ++i) - { - indices[axis] = i; - auto cur_val = input(indices.begin(), indices.end()); - if(min_val > cur_val) - { - min_val = cur_val; - min_index = i; - } - else if(select_last_index and float_equal(min_val, cur_val)) - { - min_index = i; - } - } - - return min_index; - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - std::size_t batch_item_num = args.front().get_shape().lens()[axis]; - - result.visit([&](auto output) { - args[0].visit([&](auto input) { - par_for(output_shape.elements(), [&](auto i) { - auto data_idx = output_shape.multi(i); - output[i] = this->calc_argmin(input, data_idx, batch_item_num); - }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/as_shape.hpp b/docker/rocm/migraphx/include/migraphx/op/as_shape.hpp deleted file mode 100644 index 162526de1..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/as_shape.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_AS_SHAPE_HPP -#define MIGRAPHX_GUARD_OPERATORS_AS_SHAPE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct as_shape -{ - shape s; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape")); - } - - std::string name() const { return "as_shape"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(1).standard(); - assert(inputs.front().elements() >= s.elements()); - return s; - } - argument compute(shape output_shape, std::vector args) const - { - return args.front().reshape(output_shape); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/asin.hpp b/docker/rocm/migraphx/include/migraphx/op/asin.hpp deleted file mode 100644 index b65a9ee5b..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/asin.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ASIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_ASIN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct asin : unary -{ - auto apply() const - { - return [](auto x) { return std::asin(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/asinh.hpp b/docker/rocm/migraphx/include/migraphx/op/asinh.hpp deleted file mode 100644 index 56b586630..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/asinh.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ASINH_HPP -#define MIGRAPHX_GUARD_OPERATORS_ASINH_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct asinh : unary -{ - auto apply() const - { - return [](auto x) { return std::asinh(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/atan.hpp b/docker/rocm/migraphx/include/migraphx/op/atan.hpp deleted file mode 100644 index 06a0ca535..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/atan.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ATAN_HPP -#define MIGRAPHX_GUARD_OPERATORS_ATAN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct atan : unary -{ - auto apply() const - { - return [](auto x) { return std::atan(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/atanh.hpp b/docker/rocm/migraphx/include/migraphx/op/atanh.hpp deleted file mode 100644 index 58c171fb3..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/atanh.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ATANH_HPP -#define MIGRAPHX_GUARD_OPERATORS_ATANH_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct atanh : unary -{ - auto apply() const - { - return [](auto x) { return std::atanh(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/binary.hpp b/docker/rocm/migraphx/include/migraphx/op/binary.hpp deleted file mode 100644 index 782c75ac4..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/binary.hpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_BINARY_HPP -#define MIGRAPHX_GUARD_OPERATORS_BINARY_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -template -struct binary : op_name -{ - std::string point_function() const { return this->name(); } - std::string point_op() const - { - const auto& self = static_cast(*this); - auto pf = self.point_function(); - if(pf.empty()) - return {}; - if(with_char(::ispunct)(pf.front())) - { - return "${0} " + pf + " ${1}"; - } - else - { - return "${function:" + pf + "}(${0}, ${1})"; - } - } - value base_attributes() const - { - const auto& self = static_cast(*this); - return {{"pointwise", true}, {"point_op", self.point_op()}}; - } - value attributes() const { return base_attributes(); } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, static_cast(*this), true} - .has(2) - .same_type() - .same_dims(); - auto s0 = inputs.at(0); - auto s1 = inputs.at(1); - if(s0.dynamic() or s1.dynamic()) - { - if(s0 == s1) - return s0; - MIGRAPHX_THROW("BINARY: " + point_function() + ": fixed-dyn shape for inputs"); - } - else if(s0 == s1 and s0.packed()) - { - return s0; - } - else if(s0.packed() != s1.packed()) - { - return s0.packed() ? s0 : s1; - } - else if(s0.broadcasted() != s1.broadcasted()) - { - return s0.broadcasted() ? s1.with_lens(s0.lens()) : s0.with_lens(s0.lens()); - } - else - { - return {s0.type(), s0.lens()}; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - visit_all(result, args[0], args[1])([&](auto output, auto input1, auto input2) { - par_transform(input1.begin(), - input1.end(), - input2.begin(), - output.begin(), - static_cast(*this).apply()); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/bit_cast.hpp b/docker/rocm/migraphx/include/migraphx/op/bit_cast.hpp deleted file mode 100644 index 0112342a1..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/bit_cast.hpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_BIT_CAST_HPP -#define MIGRAPHX_GUARD_OPERATORS_BIT_CAST_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Obtain a value of type `target_type` by reinterpreting - * the object represnetaion of the input. Originally used - * for casting from fp8e4m3fn to fp8e4m3fnuz. - */ -struct bit_cast : unary -{ - shape::type_t target_type; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.target_type, "target_type")); - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto input = inputs.at(0); - std::size_t target_type_size; - shape::visit(target_type, [&](auto as) { target_type_size = as.size(); }); - if(input.type_size() != target_type_size) - { - MIGRAPHX_THROW("BIT_CAST: target_type has different type_size from input's"); - } - if(input.dynamic()) - { - return {target_type, input.dyn_dims()}; - } - else - { - return {target_type, input.lens(), input.strides()}; - } - } - - std::string point_op() const - { - return "${function:bit_cast}<" + shape::cpp_type(target_type) + ">(${0})"; - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - result.visit([&](auto output) { - using otype = typename decltype(output)::value_type; - args[0].visit([&](auto input) { - using itype = typename decltype(input)::value_type; - if constexpr(sizeof(otype) == sizeof(itype)) - - { - par_transform(input.begin(), input.end(), output.begin(), [&](auto x) { - return migraphx::bit_cast(x); - }); - } - else - { - // not possible to hit this unless somehow the types change after compute_shape - // is called - MIGRAPHX_THROW("BIT_CAST: type size mismatch"); - } - }); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/bitwise_and.hpp b/docker/rocm/migraphx/include/migraphx/op/bitwise_and.hpp deleted file mode 100644 index de049df67..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/bitwise_and.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_BITWISE_AND_HPP -#define MIGRAPHX_GUARD_OPERATORS_BITWISE_AND_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct bitwise_and : binary -{ - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = true; - return a; - } - - std::string point_function() const { return "&"; } - - shape compute_shape(const std::vector& inputs) const - { - auto result = binary::compute_shape(inputs); - if(not shape::is_integral(result.type())) - { - MIGRAPHX_THROW("BITWISE_AND: only supports integral types"); - } - return result; - } - - auto apply() const - { - return [](auto x, auto y) { - if constexpr(std::is_integral{} and std::is_integral{}) - { - // NOLINTNEXTLINE(hicpp-signed-bitwise) - return x & y; - } - else - { - MIGRAPHX_THROW("BITWISE_AND: only supports integral types"); - // will never return the next line; needed for return template typing - return x; - } - }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/broadcast.hpp b/docker/rocm/migraphx/include/migraphx/op/broadcast.hpp deleted file mode 100644 index cfc2acbe6..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/broadcast.hpp +++ /dev/null @@ -1,158 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_BROADCAST_HPP -#define MIGRAPHX_GUARD_OPERATORS_BROADCAST_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * 1 input version: - * Broadcasts a tensor from the original shape to the broadcast_lens by setting the stride of - * broadcasted dimensions to zero. `axis` attribute for a 1D input shape is the output dimension - * that stays the same. - * ex: broadcasting shape [1024] -> [4, 1024, 3] has axis = 1. - * - * For higher rank input shapes, axis is an offset parameter for the broadcasting. - * Such that this operator would work in the opposite direction of NumPy broadcasting - * (left-most to rightwards element-wise comparison) - * ex: broadcasting shape [2, 2] -> [2, 2, 3] with axis = 0 - * - * 2 input version: - * Broadcast the first input 1D shape into the second input shape based on the axis parameter. - * Handles broadcasting a 1D static shape into a higher rank dynamic shape. - * broadcast_lens is not used - */ -struct broadcast -{ - uint64_t axis = 0; - std::vector broadcast_lens = {}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.broadcast_lens, "out_lens")); - } - - std::string name() const { return "broadcast"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1, 2); - auto s0 = inputs.at(0); - auto t = s0.type(); - if(inputs.size() == 1) - { - // the ONNX broadcast op is deprecated now, so not handling the negative - // value of axis anymore - if(s0.dynamic()) - MIGRAPHX_THROW( - "BROADCAST: Single dynamic input shape not supported. Use two inputs."); - if(axis >= broadcast_lens.size()) - { - MIGRAPHX_THROW("BROADCAST : axis " + migraphx::to_string(axis) + - " is out of range"); - } - if(broadcast_lens.size() - axis < s0.lens().size()) - { - MIGRAPHX_THROW("BROADCAST: (broadcast ndims - axis) is less than s0 ndims"); - } - if(not std::equal(s0.lens().begin(), s0.lens().end(), broadcast_lens.begin() + axis)) - { - MIGRAPHX_THROW("BROADCAST: when broadcasting, succeeding sizes must match"); - } - - std::vector bcast_strides(broadcast_lens.size(), 0); - std::copy(s0.strides().begin(), s0.strides().end(), bcast_strides.begin() + axis); - shape output{t, broadcast_lens, std::move(bcast_strides)}; - if(output.elements() < s0.elements()) - { - // don't think this can occur? - MIGRAPHX_THROW("BROADCAST: output size must be greater than or equal to s0 size"); - } - return output; - } - else - { - // two inputs - auto s1 = inputs.at(1); - if(s0.dynamic()) - { - MIGRAPHX_THROW("BROADCAST_2in: s0 is a dynamic shape, does not handle broadcasting " - "a dynamic shape"); - } - if(s0.ndim() != 1) - { - MIGRAPHX_THROW("BROADCAST_2in: s0 has ndim " + migraphx::to_string(s0.ndim()) + - ", only handle ndim = 1"); - } - if(axis >= s1.ndim()) - { - MIGRAPHX_THROW("BROADCAST_2in: axis " + migraphx::to_string(axis) + - " is out of range"); - } - if(s1.dynamic()) - { - s0 = s0.to_dynamic(); - if(s0.dyn_dims()[0] != s1.dyn_dims()[axis]) - { - MIGRAPHX_THROW("BROADCAST_2in: s0 length doesn't match with dynamic s1 axis " - "dimension length (" + - migraphx::to_string(s0.dyn_dims()[0]) + - " != " + migraphx::to_string(s1.dyn_dims()[axis]) + ")"); - } - return s1; - } - - if(s0.lens()[0] != s1.lens()[axis]) - { - MIGRAPHX_THROW("BROADCAST_2in: s0 length doesn't match with static s1 axis " - "dimension length (" + - migraphx::to_string(s0.lens()[0]) + - " != " + migraphx::to_string(s1.lens()[axis]) + ")"); - } - std::vector bcast_strides(s1.ndim(), 0); - std::copy(s0.strides().begin(), s0.strides().end(), bcast_strides.begin() + axis); - shape output{t, s1.lens(), std::move(bcast_strides)}; - return output; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/broadcast_for_dot.hpp b/docker/rocm/migraphx/include/migraphx/op/broadcast_for_dot.hpp deleted file mode 100644 index 5fe43685f..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/broadcast_for_dot.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_BROADCAST_FOR_DOT_HPP -#define MIGRAPHX_GUARD_OPERATORS_BROADCAST_FOR_DOT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Broadcast dimensions between two tensors for the `dot` operator. - * Essentially broadcasts between two shapes for dimensions other than the last two. - * This operator is only needed if one of the shapes are dynamic. - * Example: - * a = shape[{1, 4}, 3, 248, 248] - * b = shape[248, 365] - * broadcast_for_dot(a, b) => shape[{1, 4}, 3, 248, 248] (no change) - * broadcast_for_dot(b, a) => shape[{1, 4}, 3, 248, 365] - */ -struct broadcast_for_dot -{ - std::string name() const { return "broadcast_for_dot"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2); - check_shapes{{inputs.at(0)}, *this, true}.min_ndims(2); - check_shapes{{inputs.at(1)}, *this, true}.min_ndims(2); - auto s0 = inputs.at(0); - auto s1 = inputs.at(1); - if(s0.dynamic() or s1.dynamic()) - { - s0 = s0.to_dynamic(); - s1 = s1.to_dynamic(); - auto dds0_it = s0.dyn_dims().end() - 2; - auto dds1_it = s1.dyn_dims().end() - 2; - std::vector sliced_dds0{s0.dyn_dims().begin(), dds0_it}; - std::vector sliced_dds1{s1.dyn_dims().begin(), dds1_it}; - auto output_dyn_dims = compute_broadcasted_dyn_dims(sliced_dds0, sliced_dds1); - output_dyn_dims.insert(output_dyn_dims.end(), dds0_it, s0.dyn_dims().end()); - return {s0.type(), output_dyn_dims}; - } - else - { - auto l0_it = s0.lens().begin() + s0.ndim() - 2; - std::vector l0_broadcasted_lens(s0.lens().begin(), l0_it); - auto l1_it = s1.lens().begin() + s1.ndim() - 2; - std::vector l1_broadcasted_lens(s1.lens().begin(), l1_it); - auto output_lens = compute_broadcasted_lens(l0_broadcasted_lens, l1_broadcasted_lens); - output_lens.insert(output_lens.end(), l0_it, s0.lens().end()); - return make_bcast_shape(s0, output_lens); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/broadcast_with_dims.hpp b/docker/rocm/migraphx/include/migraphx/op/broadcast_with_dims.hpp deleted file mode 100644 index 66267008c..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/broadcast_with_dims.hpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_BROADCAST_WITH_DIMS_HPP -#define MIGRAPHX_GUARD_OPERATORS_BROADCAST_WITH_DIMS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Broadcast the input tensor to the shape defined by the values of the second input. - * Used as `broadcast_with_dims(input_tensor, dims)`, where dims is a vector of integer dimensions. - * `input_tensor` must be broadcastable with `dims`, otherwise this operator with throw at compute. - * This operator can be replaced with `multibroadcast(input_tensor)` if the `dims` vector is - * constant. - * - * Example: - * input_tensor shape: lens = {2, 3}, strides = {3, 1} - * dims = [4, 1, 3] - * output shape: lens = {4, 2, 3}, strides = {0, 3, 1} - */ -struct broadcast_with_dims -{ - std::string name() const { return "broadcast_with_dims"; } - - shape compute_shape(const std::vector& inputs) const - { - migraphx::check_shapes{inputs, *this, true}.has(2); - // check that second input has a static shape - (void)migraphx::check_shapes{inputs.begin() + 1, inputs.end(), *this, false}; - // output tensor rank is greater of input_tensor rank or length of dims vector - const auto& input_tensor_shape = inputs.at(0); - const auto& dims_shape = inputs.at(1); - size_t out_ndim = std::max(input_tensor_shape.ndim(), dims_shape.lens().at(0)); - std::size_t max_int = std::numeric_limits::max(); - std::vector dyn_dims(out_ndim, - shape::dynamic_dimension{0, max_int}); - return {input_tensor_shape.type(), dyn_dims}; - } - - argument compute(const shape& output_shape, const std::vector& args) const - { - auto s0 = args.at(0).get_shape(); - const auto& in_lens = s0.lens(); - std::vector dims_input(output_shape.ndim()); - args.at(1).visit([&](auto a) { dims_input.assign(a.begin(), a.end()); }); - auto out_lens = compute_broadcasted_lens(in_lens, dims_input); - auto out_shape = make_bcast_shape(s0, out_lens); - return args[0].reshape(out_shape); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/capture.hpp b/docker/rocm/migraphx/include/migraphx/op/capture.hpp deleted file mode 100644 index efb71c5c3..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/capture.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CAPTURE_HPP -#define MIGRAPHX_GUARD_OPERATORS_CAPTURE_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct capture -{ - std::size_t ins_index; - std::function)> f{}; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.ins_index, "ins_index")); - } - - std::string name() const { return "capture"; } - - shape compute_shape(std::vector inputs) const { return inputs.front(); } - - // the context argument is added to prevent the op from be eliminated by - // constant propagation - argument compute(context&, const shape&, const std::vector& args) const - { - if(f) - { - f(ins_index, args); - } - else - { - MIGRAPHX_THROW("CAPTURE: callback function is not callable!"); - } - - return args.front(); - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/ceil.hpp b/docker/rocm/migraphx/include/migraphx/op/ceil.hpp deleted file mode 100644 index fcf41c02d..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/ceil.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CEIL_HPP -#define MIGRAPHX_GUARD_OPERATORS_CEIL_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct ceil : unary -{ - auto apply() const - { - return [](auto x) { return std::ceil(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/clip.hpp b/docker/rocm/migraphx/include/migraphx/op/clip.hpp deleted file mode 100644 index 83ad25239..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/clip.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CLIP_HPP -#define MIGRAPHX_GUARD_OPERATORS_CLIP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct clip -{ - std::string name() const { return "clip"; } - - value attributes() const - { - return {{"pointwise", true}, - {"point_op", "${function:min}(${function:max}(${1}, ${0}), ${2})"}}; - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(3).same_type().same_dims(); - return inputs.front(); - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - visit_all(result, args[0], args[1], args[2])([&](auto output, auto x, auto min, auto max) { - par_for(dyn_out.computed_shape.elements(), - [&](auto i) { output[i] = std::min(std::max(min[i], x[i]), max[i]); }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/common.hpp b/docker/rocm/migraphx/include/migraphx/op/common.hpp deleted file mode 100644 index e6b85f19e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/common.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_COMMON_HPP -#define MIGRAPHX_GUARD_OPERATORS_COMMON_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -// Specifies where to add the "extra" cell of padding if the -// calculated padding is an odd number. -// Padding mode is default_ for fixed shape padding. -// same_lower and same_upper specify dynamic padding. -// The odd cell goes at the beginning of the dimension -// (same_lower) or end (same_upper). -enum padding_mode_t -{ - default_, // NOLINT - same_lower, - same_upper -}; - -// The pooling modes must correspond 1-1 to the operators defined for struct parse_pooling. -// Used in pooling and roialign operators. -enum class pooling_mode -{ - average, - max, - lpnorm -}; - -// indicate rnn computation direction -enum class rnn_direction -{ - forward, - reverse, - bidirectional, -}; - -MIGRAPHX_EXPORT std::ostream& operator<<(std::ostream& os, pooling_mode v); -MIGRAPHX_EXPORT std::ostream& operator<<(std::ostream& os, rnn_direction v); - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/concat.hpp b/docker/rocm/migraphx/include/migraphx/op/concat.hpp deleted file mode 100644 index 527ef55bc..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/concat.hpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CONCAT_HPP -#define MIGRAPHX_GUARD_OPERATORS_CONCAT_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct concat -{ - int64_t axis = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "concat"; } - std::vector compute_offsets(const shape& output_shape, - const std::vector& args) const - { - auto n_dims = args[0].get_shape().lens().size(); - std::vector offsets; - std::vector offset(n_dims, 0); - offset[axis] = 0; - for(const auto& arg : args) - { - offsets.push_back(output_shape.index(offset)); - offset[axis] += arg.get_shape().lens()[axis]; - } - return offsets; - } - - shape normalize_compute_shape(std::vector inputs) const - { - // inputs can contain 1 or more shapes (variadic). compute_shape_op ensures there must - // be at least 1. - check_shapes{inputs, *this, true}.same_ndims().same_type(); - - if(std::none_of(inputs.begin(), inputs.end(), [&](const shape& s) { return s.dynamic(); })) - { - // Static input shapes - const auto& first_shape_lens = inputs.front().lens(); - const auto& type = inputs.front().type(); - for(std::size_t ll = 0; ll < first_shape_lens.size(); ll++) - { - if(ll != axis) - { - if(not std::all_of(inputs.begin(), inputs.end(), [&](auto s) { - return s.lens()[ll] == first_shape_lens[ll]; - })) - { - MIGRAPHX_THROW("CONCAT: all input dimensions should match along axis " + - std::to_string(ll)); - } - } - } - std::size_t new_dim_axis = 0; - for(const auto& input : inputs) - { - const auto& lens = input.lens(); - new_dim_axis += lens[axis]; - } - std::vector new_lens = first_shape_lens; - new_lens[axis] = new_dim_axis; - return shape::from_permutation(type, new_lens, find_permutation(inputs)); - } - else if(std::all_of( - inputs.begin(), inputs.end(), [&](const shape& s) { return s.dynamic(); })) - { - // Dynamic input shapes - for(std::size_t index = 0; index < inputs[0].ndim(); index++) - { - if(index != axis) - { - if(not std::all_of(inputs.begin(), inputs.end(), [&](const shape& s) { - return s.dyn_dims()[index] == inputs[0].dyn_dims()[index]; - })) - MIGRAPHX_THROW("CONCAT: all input dimensions should match in axis " + - std::to_string(index)); - } - } - std::size_t new_min = 0; - std::size_t new_max = 0; - for(const auto& input : inputs) - { - auto ddim = input.dyn_dims()[axis]; - new_min += ddim.min; - new_max += ddim.max; - } - - auto new_dims = inputs[0].dyn_dims(); - new_dims[axis] = migraphx::shape::dynamic_dimension{new_min, new_max}; - return {inputs[0].type(), new_dims}; - } - else - { - MIGRAPHX_THROW("CONCAT: Cannot mix static and dynamic input shapes."); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - std::vector coffsets = compute_offsets(dyn_out.computed_shape, args); - for(std::size_t l = 0; l < args.size(); l++) - { - auto argl = args[l]; - visit_all(result, argl)([&](auto output, auto input) { - auto slice_shape = shape{dyn_out.computed_shape.type(), - input.get_shape().lens(), - dyn_out.computed_shape.strides()}; - auto slice = make_view(slice_shape, output.data() + coffsets[l]); - std::copy(input.begin(), input.end(), slice.begin()); - }); - } - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/contiguous.hpp b/docker/rocm/migraphx/include/migraphx/op/contiguous.hpp deleted file mode 100644 index 695869aa5..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/contiguous.hpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CONTIGUOUS_HPP -#define MIGRAPHX_GUARD_OPERATORS_CONTIGUOUS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/// The contiguous operator takes a non-standard input tensor and returns -/// the same tensor but in standard form. For example, if input tensor A which has lens = (4,5) -/// is first transposed, i.e. lens = (5,4), this tensor's data layout remained the same -/// during the transpose operation; only it's shape lengths and strides were changed. -/// This leaves the tensor in a non-standard form. The contiguous operator copies the -/// underlying data such that resulting tensor is returned to a standard form. -struct contiguous -{ - std::string name() const { return "contiguous"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto s0 = inputs.front(); - if(s0.dynamic()) - { - return s0; - } - else - { - const auto& lens = s0.lens(); - auto t = s0.type(); - return {t, lens}; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - assert(dyn_out.computed_shape.standard()); - argument result{dyn_out.computed_shape}; - visit_all(result, args[0])([&](auto output, auto input) { - shape_for_each(output.get_shape(), [&](const auto& idx) { - output(idx.begin(), idx.end()) = input(idx.begin(), idx.end()); - }); - }); - return result; - } - - auto apply() const - { - return [](auto x) { return x; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/convert.hpp b/docker/rocm/migraphx/include/migraphx/op/convert.hpp deleted file mode 100644 index 22f9c2d81..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/convert.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CONVERT_HPP -#define MIGRAPHX_GUARD_OPERATORS_CONVERT_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct convert : unary -{ - shape::type_t target_type = shape::half_type; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.target_type, "target_type")); - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto input = inputs.at(0); - if(input.dynamic()) - { - return {target_type, input.dyn_dims()}; - } - else - { - return {target_type, input.lens(), input.strides()}; - } - } - - std::string point_op() const - { - return "${function:convert}<" + shape::cpp_type(target_type) + ">(${0})"; - } - - auto apply() const - { - auto type = target_type; - return [type](auto x) { - auto y = x; - shape::visit(type, [&](auto as) { - // clamping value between target_type's max and min doesn't work for NaNs, - if(std::isnan(static_cast(x))) - { - y = as.nan(); - } - else if(shape::is_integral(type) and std::is_floating_point_v) - { - // for the floating point to integer conversion, clamp first and then convert to - // avoid undefined behaviour - y = as(std::min(std::max(static_cast(x), static_cast(as.min())), - static_cast(as.max()))); - } - else - { - // clamp overflowing/underflowing values to min()/max() instead of +/-infinity - // during downcasting - y = std::min(std::max(as(x), as.min()), as.max()); - } - }); - return y; - }; - } - - convert(shape::type_t t) : target_type{t} {} - convert() {} -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/convolution.hpp b/docker/rocm/migraphx/include/migraphx/op/convolution.hpp deleted file mode 100644 index 436d8c2d3..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/convolution.hpp +++ /dev/null @@ -1,242 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CONVOLUTION_HPP -#define MIGRAPHX_GUARD_OPERATORS_CONVOLUTION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Convolution operator. Does not support optimal dimensions for spatial dimensions. Returns empty - * optimals. - */ -struct convolution -{ - std::vector padding = {0, 0}; - std::vector stride = {1, 1}; - std::vector dilation = {1, 1}; - - int group = 1; - padding_mode_t padding_mode = default_; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.padding, "padding"), - f(self.stride, "stride"), - f(self.dilation, "dilation"), - f(self.group, "group"), - f(self.padding_mode, "padding_mode")); - } - - std::string name() const { return "convolution"; } - - void check_attribute_size() const - { - if((padding.size() != stride.size() and (padding.size() / 2) != stride.size()) or - stride.size() != dilation.size()) - { - MIGRAPHX_THROW("CONVOLUTION: inconsistent attribute sizes"); - } - } - - value attributes() const { return {{"normalize_padding", "padding"}}; } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2).same_type().same_ndims().min_ndims(3); - check_attribute_size(); - // num of dims of input and attribute should match - const auto input_ndim = inputs[0].ndim(); - const auto padding_size = padding.size(); - - if(input_ndim != padding_size / 2 + 2 and input_ndim != padding_size + 2) - { - MIGRAPHX_THROW("CONVOLUTION: input and attribute size mismatch!"); - } - - const shape& x_shape = inputs.at(0); - const shape& w_shape = inputs.at(1); - const size_t num_spatial_dims = input_ndim - 2; - if(num_spatial_dims != this->kdims()) - { - MIGRAPHX_THROW("CONVOLUTION: input k-dims does not match attribute size"); - } - - if(not x_shape.dynamic() and not w_shape.dynamic() and - x_shape.lens().at(1) != (w_shape.lens().at(1) * group)) - MIGRAPHX_THROW("CONVOLUTION: mismatched channel numbers"); - - if(x_shape.dynamic() or w_shape.dynamic()) - { - return dynamic_compute_shape(x_shape, w_shape); - } - else - { - return static_compute_shape(x_shape, w_shape); - } - } - - std::vector calc_conv_lens(std::vector x_lens, - std::vector w_lens) const - { - const size_t num_spatial_dims = x_lens.size() - 2; - std::vector ret = {}; - // calculate the output shape of the convolution: ((W - K + 2P) / S) + 1 - for(size_t i = 0; i < num_spatial_dims; i++) - { - if(x_lens[i] == 0 or w_lens[i] == 0) - { - // for handling when a dimension = 0 (opt of dynamic_dimension) - ret.push_back(0); - } - else - { - auto padding_factor = 2 * padding[i]; - if(padding.size() == 2 * num_spatial_dims) - { - // when padding is {x0_begin, x1_begin, ... x0_end , x1_end, ...} - padding_factor = padding[i] + padding[i + num_spatial_dims]; - } - ret.push_back(std::size_t(std::max( - 1, - (x_lens[i + 2] - (1 + dilation[i] * (w_lens[i + 2] - 1)) + padding_factor) / - stride[i] + - 1))); - } - } - return ret; - } - - shape dynamic_compute_shape(shape x_shape, shape w_shape) const - { - std::vector output_dyn_dims = {}; - output_dyn_dims.push_back(x_shape.to_dynamic().dyn_dims().at(0)); - output_dyn_dims.push_back(w_shape.to_dynamic().dyn_dims().at(0)); - - const size_t num_spatial_dims = x_shape.ndim() - 2; - if(padding_mode != default_) - { - for(std::size_t i = 0; i < num_spatial_dims; ++i) - { - auto ceil_div = [](std::size_t x, std::size_t y) { return (x + y - 1) / y; }; - auto s = stride[i]; - if(x_shape.dynamic()) - { - auto x = x_shape.dyn_dims()[i + 2]; - std::set optimals{}; - std::transform(x.optimals.begin(), - x.optimals.end(), - std::inserter(optimals, optimals.begin()), - [&](auto o) { return ceil_div(o, s); }); - output_dyn_dims.push_back( - shape::dynamic_dimension{ceil_div(x.min, s), ceil_div(x.max, s), optimals}); - } - else - { - auto od = ceil_div(x_shape.lens()[i + 2], s); - output_dyn_dims.push_back(shape::dynamic_dimension{od, od}); - } - } - } - else - { - // Does not compute for optimals - auto min_spatial_dims = calc_conv_lens(x_shape.min_lens(), w_shape.max_lens()); - auto max_spatial_dims = calc_conv_lens(x_shape.max_lens(), w_shape.min_lens()); - for(size_t i = 0; i < num_spatial_dims; ++i) - { - output_dyn_dims.push_back( - shape::dynamic_dimension{min_spatial_dims[i], max_spatial_dims[i], {}}); - } - } - return shape{x_shape.type(), output_dyn_dims}; - } - - shape static_compute_shape(shape x_shape, shape w_shape) const - { - std::vector output_lens{x_shape.lens()[0], w_shape.lens()[0]}; - auto spatial_lens = calc_conv_lens(x_shape.lens(), w_shape.lens()); - std::for_each(spatial_lens.begin(), spatial_lens.end(), [&output_lens](auto x) { - output_lens.push_back(x); - }); - return x_shape.with_lens(output_lens); - } - - size_t kdims() const - { - check_attribute_size(); - return stride.size(); - } - - argument compute(shape output_shape, std::vector args) const - { - std::vector new_padding; - if(padding_mode != op::padding_mode_t::default_) - { - // auto-Calculate the padding sizes with calc_dyn_auto_pad - auto input_lens = args[0].get_shape().lens(); - auto weights_lens = args[1].get_shape().lens(); - new_padding = - padding_mode == op::same_upper - ? calc_dyn_auto_pad(input_lens, weights_lens, stride, dilation, true) - : calc_dyn_auto_pad(input_lens, weights_lens, stride, dilation, false); - output_shape = compute_padded_shape( - args[0].get_shape(), args[1].get_shape(), new_padding, stride, dilation); - } - else - { - // Use the padding that was given - new_padding = padding; - if(output_shape.dynamic()) - { - output_shape = - normalize_compute_shape({args.at(0).get_shape(), args.at(1).get_shape()}); - } - } - - argument result{output_shape}; - visit_all(result, args[0], args[1])([&](auto output, auto input, auto weights) { - migraphx::convolution(output, input, weights, new_padding, stride, dilation, group); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/convolution_backwards.hpp b/docker/rocm/migraphx/include/migraphx/op/convolution_backwards.hpp deleted file mode 100644 index 914551836..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/convolution_backwards.hpp +++ /dev/null @@ -1,225 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_CONVOLUTION_BACKWARDS_HPP -#define MIGRAPHX_GUARD_OPERATORS_CONVOLUTION_BACKWARDS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct convolution_backwards -{ - std::vector padding = {0, 0}; - std::vector stride = {1, 1}; - std::vector dilation = {1, 1}; - - padding_mode_t padding_mode = default_; - int group = 1; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.padding, "padding"), - f(self.stride, "stride"), - f(self.dilation, "dilation"), - f(self.padding_mode, "padding_mode"), - f(self.group, "group")); - } - - std::string name() const { return "convolution_backwards"; } - - void check_attribute_size() const - { - if(padding.size() != stride.size() or stride.size() != dilation.size()) - { - MIGRAPHX_THROW("CONVOLUTION_BACKWARDS: inconsistent attribute sizes"); - } - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2).same_type().same_ndims().min_ndims(3); - - const shape& x_shape = inputs.at(0); - const shape& w_shape = inputs.at(1); - if(x_shape.ndim() - 2 != this->kdims()) - { - MIGRAPHX_THROW("CONVOLUTION_BACKWARDS: input k-dims does not match attribute size"); - } - - if(not x_shape.dynamic() and not w_shape.dynamic() and - x_shape.lens().at(1) != (w_shape.lens().at(0) * group)) - { - MIGRAPHX_THROW("CONVOLUTION_BACKWARDS: mismatched channel numbers"); - } - - if(x_shape.dynamic() or w_shape.dynamic()) - { - return dynamic_compute_shape(x_shape, w_shape); - } - else - { - return static_compute_shape(x_shape, w_shape); - } - } - - std::vector calc_spatial_lens(std::vector x_lens, - std::vector w_lens) const - { - std::vector spatial_lens(x_lens.size() - 2); - - // stride * (input - 1) + output_padding + ((kernel - 1) * dilation + 1) - padding_L - - // padding_R. This assumes padding_L = padding_R and output_padding handled in parser. - for(size_t i = 0; i < spatial_lens.size(); i++) - { - spatial_lens.at(i) = (std::size_t(std::max( - 1, - stride[i] * (x_lens[i + 2] - 1) + ((w_lens[i + 2] - 1) * dilation[i] + 1) - - 2 * padding[i]))); - } - return spatial_lens; - } - - shape dynamic_compute_shape(shape x_shape, shape w_shape) const - { - std::vector output_dyn_dims = {}; - output_dyn_dims.push_back(x_shape.to_dynamic().dyn_dims().at(0)); - output_dyn_dims.push_back(w_shape.to_dynamic().dyn_dims().at(1)); - const std::size_t num_spatial_dims = x_shape.ndim() - 2; - // Does not compute for optimals - auto min_spatial_dims = calc_spatial_lens(x_shape.min_lens(), w_shape.min_lens()); - auto max_spatial_dims = calc_spatial_lens(x_shape.max_lens(), w_shape.max_lens()); - for(size_t i = 0; i < num_spatial_dims; ++i) - { - output_dyn_dims.push_back( - shape::dynamic_dimension{min_spatial_dims[i], max_spatial_dims[i], {}}); - } - return shape{x_shape.type(), output_dyn_dims}; - } - - shape static_compute_shape(shape x_shape, shape w_shape) const - { - std::vector output_lens{x_shape.lens()[0], w_shape.lens()[1]}; - auto spatial_lens = calc_spatial_lens(x_shape.lens(), w_shape.lens()); - std::for_each(spatial_lens.begin(), spatial_lens.end(), [&output_lens](auto x) { - output_lens.push_back(x); - }); - return x_shape.with_lens(output_lens); - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - auto num_spatial_dims = this->kdims(); - visit_all(result, args[0], args[1])([&](auto output, auto input, auto weights) { - using type = typename decltype(output)::value_type; - - std::fill(output.begin(), output.end(), type{0}); - - auto in_lens = input.get_shape().lens(); - auto in_n = in_lens[0]; - auto in_c = in_lens[1]; - - auto wei = weights.get_shape().lens(); - auto wei_n = wei[0]; - auto wei_c = wei[1]; - - auto out_lens = dyn_out.computed_shape.lens(); - - std::vector win_size{in_c}; - std::copy(in_lens.begin() + 2, in_lens.end(), std::back_inserter(win_size)); - std::copy(wei.begin() + 2, wei.end(), std::back_inserter(win_size)); - shape win_shape{dyn_out.computed_shape.type(), win_size}; - - par_dfor(in_n, wei_c)([&](int o, int k) { - shape_for_each(win_shape, [&](const auto& idx_win) { - const int w = idx_win[0]; - - auto input_dims_start = idx_win.begin() + 1; - auto wei_dims_start = idx_win.begin() + num_spatial_dims + 1; - - std::vector win_start; - for(std::size_t n = 0; n < num_spatial_dims; ++n) - { - win_start.push_back(std::ptrdiff_t(*(input_dims_start + n) * stride[n]) - - std::ptrdiff_t(padding[n])); - } - - const int group_id = w / (wei_n / group); - const int in_ch = group_id * wei_c + k; - - std::vector idx_out{o, in_ch}; - - for(size_t n = 0; n < num_spatial_dims; n++) - { - idx_out.push_back(win_start[n] + *(wei_dims_start + n) * dilation[n]); - } - - std::vector idx_wei{w, k}; - std::copy(wei_dims_start, idx_win.end(), std::back_inserter(idx_wei)); - - std::vector idx_in{o, w}; - std::copy(input_dims_start, wei_dims_start, std::back_inserter(idx_in)); - - if(std::all_of( - idx_out.begin() + 2, idx_out.end(), [&](auto ii) { return ii >= 0; }) and - std::equal(idx_out.begin() + 2, - idx_out.end(), - out_lens.begin() + 2, - out_lens.end(), - std::less{})) - { - output(idx_out.begin(), idx_out.end()) += - input(idx_in.begin(), idx_in.end()) * - weights(idx_wei.begin(), idx_wei.end()); - } - }); - }); - }); - return result; - } - - size_t kdims() const - { - check_attribute_size(); - return stride.size(); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/cos.hpp b/docker/rocm/migraphx/include/migraphx/op/cos.hpp deleted file mode 100644 index 6bbfe29b9..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/cos.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_COS_HPP -#define MIGRAPHX_GUARD_OPERATORS_COS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct cos : unary -{ - auto apply() const - { - return [](auto x) { return std::cos(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/cosh.hpp b/docker/rocm/migraphx/include/migraphx/op/cosh.hpp deleted file mode 100644 index edf4316e9..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/cosh.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_COSH_HPP -#define MIGRAPHX_GUARD_OPERATORS_COSH_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct cosh : unary -{ - auto apply() const - { - return [](auto x) { return std::cosh(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/dequantizelinear.hpp b/docker/rocm/migraphx/include/migraphx/op/dequantizelinear.hpp deleted file mode 100644 index 60500b168..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/dequantizelinear.hpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_DEQUANTIZE_LINEAR_HPP -#define MIGRAPHX_GUARD_OPERATORS_DEQUANTIZE_LINEAR_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct dequantizelinear -{ - - value attributes() const - { - // Note: point_op attribute is not used in this op. Instead, in - // gpu compilation pipeline, rewrite_quantization will be invoked - // from generate_pointwise() to rewrite this op. - return {{"pointwise", true}}; - } - - std::string name() const { return "dequantizelinear"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.same_dims().has(2, 3); - if(inputs.size() == 3 and inputs[0].type() != inputs[2].type()) - { - MIGRAPHX_THROW("DEQUANTIZELINEAR: Zero point and input should be the same type."); - } - return inputs[0].with_lens(inputs[1].type(), inputs[0].lens()); - } - - argument compute(const shape& output_shape, std::vector args) const - { - auto x = args.at(0); - auto x_scale = args.at(1); - std::vector zeros(output_shape.bytes(), 0); - argument x_zero_point{{x.get_shape().type(), output_shape.lens()}, zeros.data()}; - if(args.size() == 3) - { - x_zero_point = args.at(2); - } - - argument result{output_shape}; - visit_all(x, x_zero_point)([&](auto input, auto zero_pts) { - visit_all(result, x_scale)([&](auto output, auto scales) { - par_for(output_shape.elements(), [&](auto i) { - output[i] = static_cast(static_cast(input[i]) - - static_cast(zero_pts[i])) * - scales[i]; - }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/dimensions_of.hpp b/docker/rocm/migraphx/include/migraphx/op/dimensions_of.hpp deleted file mode 100644 index 1a02313f9..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/dimensions_of.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_DIMENSIONS_OF_HPP -#define MIGRAPHX_GUARD_OPERATORS_DIMENSIONS_OF_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Returns the dimensions of the input argument from starting axis to ending axis. - * Atleast `end` must be set to use this operator (set `end` to ndim for default ONNX behavior of - * `Shape` operator) This should only be used for dynamic shapes as this can be simplified to a - * literal for static shapes. - */ -struct dimensions_of -{ - std::size_t start = 0; - std::size_t end = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.start, "start"), f(self.end, "end")); - } - - std::string name() const { return "dimensions_of"; } - - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this, true}.has(1); - if(start >= end) - { - MIGRAPHX_THROW("DIMENSIONS_OF: start >= end. start = " + std::to_string(start) + - ", end = " + std::to_string(end)); - } - return shape{shape::int64_type, {end - start}}; - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - auto input_lens = args[0].get_shape().lens(); - result.visit([&](auto output) { - std::copy(input_lens.cbegin() + start, input_lens.cbegin() + end, output.begin()); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/div.hpp b/docker/rocm/migraphx/include/migraphx/op/div.hpp deleted file mode 100644 index f32b35c45..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/div.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_DIV_HPP -#define MIGRAPHX_GUARD_OPERATORS_DIV_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct div : binary

-{ - std::string point_function() const { return "/"; } - auto apply() const - { - return [](auto x, auto y) { return x / y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/dot.hpp b/docker/rocm/migraphx/include/migraphx/op/dot.hpp deleted file mode 100644 index 37e38f762..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/dot.hpp +++ /dev/null @@ -1,139 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_DOT_HPP -#define MIGRAPHX_GUARD_OPERATORS_DOT_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Matrix multiplication of two tensors. - */ -struct dot -{ - std::string name() const { return "dot"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.same_type().same_ndims().has(2); - const shape& a = inputs.at(0); - const shape& b = inputs.at(1); - auto t = a.type(); - - if(not std::all_of(inputs.begin(), inputs.end(), [](auto s) { return s.ndim() >= 2; })) - { - MIGRAPHX_THROW("DOT: dot only accepts operands with 2 or more dimensions "); - } - if(a.dynamic() or b.dynamic()) - { - auto s0 = a.to_dynamic(); - auto s1 = b.to_dynamic(); - std::vector out_dyn_dims; - - // Check outer dynamic dimensions are compatible. - // Must allow for intersection because of how simplify_dyn_ops - // simplifies each broadcast_for_dot individually. - bool same_outers = std::equal(s0.dyn_dims().begin(), - s0.dyn_dims().end() - 2, - s1.dyn_dims().begin(), - s1.dyn_dims().end() - 2, - [&](auto x, auto y) { - auto intersect = x.intersection(y); - if(intersect.has_value()) - { - out_dyn_dims.push_back(intersect.value()); - return true; - } - return false; - }); - - if(not same_outers) - { - MIGRAPHX_THROW("DOT: dynamic outer dimensions of A and B are not compatible: {" + - to_string_range(s0.dyn_dims()) + "} x {" + - to_string_range(s1.dyn_dims()) + "}"); - } - std::size_t dim_i = s0.ndim() - 2; - std::size_t dim_j = s0.ndim() - 1; - auto x = s0.dyn_dims()[dim_j]; - auto y = s1.dyn_dims()[dim_i]; - - // check inner dimensions are compatible - if(not x.intersection(y).has_value()) - { - MIGRAPHX_THROW("DOT: dynamic inner dimensions are not compatible: {" + - to_string_range(s0.dyn_dims()) + "} x {" + - to_string_range(s1.dyn_dims()) + "}"); - } - - out_dyn_dims.push_back(s0.dyn_dims()[dim_i]); - out_dyn_dims.push_back(s1.dyn_dims()[dim_j]); - return {t, out_dyn_dims}; - } - else - { - // only handle the case that all the dimensions except the last two are the same - if(not std::equal( - a.lens().rbegin() + 2, a.lens().rend(), b.lens().rbegin() + 2, b.lens().rend())) - { - MIGRAPHX_THROW("DOT: static outer dimensions of A and B mismatch: {" + - to_string_range(a.lens()) + "} x {" + to_string_range(b.lens()) + - "}"); - } - - std::size_t dim_0 = a.ndim() - 2; - std::size_t dim_1 = a.ndim() - 1; - if(a.lens()[dim_1] != b.lens()[dim_0]) - { - MIGRAPHX_THROW("DOT: static inner dimensions do not match: {" + - to_string_range(a.lens()) + "} x {" + to_string_range(b.lens()) + - "}"); - } - - auto out_lens = a.lens(); - out_lens[dim_1] = b.lens()[dim_1]; - return {t, out_lens}; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result = argument{dyn_out.computed_shape}; - visit_all(result, args[0], args[1])( - [&](auto cmat, auto amat, auto bmat) { gemm(cmat, amat, bmat, 1.0f, 0.0f); }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/elu.hpp b/docker/rocm/migraphx/include/migraphx/op/elu.hpp deleted file mode 100644 index 3f0eddb61..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/elu.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ELU_HPP -#define MIGRAPHX_GUARD_OPERATORS_ELU_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct elu : unary -{ - float alpha = 1; - - std::string point_op() const - { - return "${function:where}(${0} > 0, ${0}, ${alpha} * (${function:exp}(${0}) - 1))"; - } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.alpha, "alpha")); - } - - auto apply() const - { - return [&](auto x) { return x > 0 ? x : alpha * std::expm1(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/equal.hpp b/docker/rocm/migraphx/include/migraphx/op/equal.hpp deleted file mode 100644 index 005ab4320..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/equal.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_EQUAL_HPP -#define MIGRAPHX_GUARD_OPERATORS_EQUAL_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct equal : binary -{ - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = true; - return a; - } - std::string point_function() const { return "=="; } - auto apply() const - { - return [](auto x, auto y) { return float_equal(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/erf.hpp b/docker/rocm/migraphx/include/migraphx/op/erf.hpp deleted file mode 100644 index 69b0a8db5..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/erf.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ERF_HPP -#define MIGRAPHX_GUARD_OPERATORS_ERF_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct erf : unary -{ - auto apply() const - { - return [](auto x) { return std::erf(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/exp.hpp b/docker/rocm/migraphx/include/migraphx/op/exp.hpp deleted file mode 100644 index 0fe8fc210..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/exp.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_EXP_HPP -#define MIGRAPHX_GUARD_OPERATORS_EXP_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct exp : unary -{ - auto apply() const - { - return [](auto x) { return std::exp(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/fill.hpp b/docker/rocm/migraphx/include/migraphx/op/fill.hpp deleted file mode 100644 index 38d608b91..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/fill.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_FILL_HPP -#define MIGRAPHX_GUARD_OPERATORS_FILL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * fill(default_value, output_buffer) - * Fill an output buffer with the given default_value. - * Note that if the default_value is a literal and the output_buffer - * has a static shape this operator can be replaced with a literal. - */ -struct fill -{ - std::string name() const { return "fill"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2).same_type(); - if(inputs.at(0).dynamic() or inputs.at(0).elements() != 1) - { - MIGRAPHX_THROW("FILL: default_value is dynamic or more than one element"); - } - return inputs.back(); - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - visit_all(args[0], args[1])([&](auto value, auto output) { - par_for(dyn_out.computed_shape.elements(), [&](auto i) { output[i] = value.front(); }); - }); - return args[1]; - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 1; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/flatten.hpp b/docker/rocm/migraphx/include/migraphx/op/flatten.hpp deleted file mode 100644 index 55fab0d33..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/flatten.hpp +++ /dev/null @@ -1,107 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_FLATTEN_HPP -#define MIGRAPHX_GUARD_OPERATORS_FLATTEN_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct flatten -{ - int64_t axis = 1; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = - value::array{normalize_attribute::include_min, normalize_attribute::include_max}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "flatten"; } - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto s = inputs[0]; - if(s.dynamic()) - { - // Doesn't handle optimals - auto min_lens = s.min_lens(); - auto max_lens = s.max_lens(); - // If any of the opt values is 0, output opt will be 0 - shape::dynamic_dimension x = { - std::accumulate( - min_lens.begin(), min_lens.begin() + axis, std::size_t{1}, std::multiplies<>{}), - std::accumulate( - max_lens.begin(), max_lens.begin() + axis, std::size_t{1}, std::multiplies<>{}), - {}}; - shape::dynamic_dimension y = { - std::accumulate( - min_lens.begin() + axis, min_lens.end(), std::size_t{1}, std::multiplies<>{}), - std::accumulate( - max_lens.begin() + axis, max_lens.end(), std::size_t{1}, std::multiplies<>{}), - {}}; - return {s.type(), {x, y}}; - } - else - { - auto&& lens = s.lens(); - auto x = std::accumulate( - lens.begin(), lens.begin() + axis, std::size_t{1}, std::multiplies<>{}); - auto y = std::accumulate( - lens.begin() + axis, lens.end(), std::size_t{1}, std::multiplies<>{}); - return {s.type(), {x, y}}; - } - } - argument compute(const dyn_output& dyn_out, std::vector args) const - { - assert(dyn_out.computed_shape.standard()); - argument result{dyn_out.computed_shape}; - - visit_all(result, args[0])([&](auto output, auto input) { - std::copy(input.begin(), input.end(), output.begin()); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/floor.hpp b/docker/rocm/migraphx/include/migraphx/op/floor.hpp deleted file mode 100644 index 38100bd8d..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/floor.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_FLOOR_HPP -#define MIGRAPHX_GUARD_OPERATORS_FLOOR_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct floor : unary -{ - auto apply() const - { - return [](auto x) { return std::floor(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/fmod.hpp b/docker/rocm/migraphx/include/migraphx/op/fmod.hpp deleted file mode 100644 index abae2589d..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/fmod.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_FMOD_HPP -#define MIGRAPHX_GUARD_OPERATORS_FMOD_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct fmod : binary -{ - std::string name() const { return "fmod"; } - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = false; - return a; - } - auto apply() const - { - return [](auto x, auto y) { return std::fmod(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/gather.hpp b/docker/rocm/migraphx/include/migraphx/op/gather.hpp deleted file mode 100644 index b90d88550..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/gather.hpp +++ /dev/null @@ -1,149 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_GATHER_HPP -#define MIGRAPHX_GUARD_OPERATORS_GATHER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct gather -{ - int64_t axis = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "gather"; } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2); - shape data = inputs[0]; - shape indices = inputs[1]; - auto type = data.type(); - // If index_dims is dynamic, convert the data to dynamic too. - if(indices.dynamic()) - { - data = data.to_dynamic(); - } - if(data.dynamic()) - { - auto dims = data.dyn_dims(); - dims.erase(dims.begin() + axis); - - if(not indices.scalar()) - { - auto index_dims = indices.to_dynamic().dyn_dims(); - dims.insert(dims.begin() + axis, index_dims.begin(), index_dims.end()); - } - return {type, dims}; - } - else - { - // Both data and indices are static. indices may be scalar - auto lens = data.lens(); - lens.erase(lens.begin() + axis); - - if(not indices.scalar()) - { - auto ind_lens = indices.lens(); - lens.insert(lens.begin() + axis, ind_lens.begin(), ind_lens.end()); - } - - // for scalar output - if(lens.empty()) - { - return {type}; - } - - return {type, lens}; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - // negative axis means counting dimensions from back - auto lens = args[0].get_shape().lens(); - std::size_t axis_dim_size = lens[axis]; - // max dimension in axis - visit_all(result, args[0])([&](auto output, auto data) { - args[1].visit([&](auto indices) { - if(dyn_out.computed_shape.scalar()) - { - auto in_index = indices.front(); - in_index = (in_index < 0) ? in_index + axis_dim_size : in_index; - output[0] = data[in_index]; - } - else - { - auto out_lens = data.get_shape().lens(); - out_lens[axis] = indices.get_shape().elements(); - migraphx::shape out_comp_shape{data.get_shape().type(), out_lens}; - shape_for_each(out_comp_shape, [&](const auto& out_idx_v, size_t out_idx) { - auto data_idx = out_idx_v; - auto in_index = indices[data_idx[axis]]; - in_index = (in_index < 0) ? in_index + axis_dim_size : in_index; - // don't go out of bounds: https://github.com/ROCm/AMDMIGraphX/issues/2838 - assert(in_index >= 0 and in_index < axis_dim_size); - data_idx[axis] = in_index; - output[out_idx] = data(data_idx.begin(), data_idx.end()); - }); - } - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/gathernd.hpp b/docker/rocm/migraphx/include/migraphx/op/gathernd.hpp deleted file mode 100644 index 2c4177480..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/gathernd.hpp +++ /dev/null @@ -1,226 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_GATHERND_HPP -#define MIGRAPHX_GUARD_OPERATORS_GATHERND_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct gathernd -{ - int batch_dims = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.batch_dims, "batch_dims")); - } - - std::string name() const { return "gathernd"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2); - auto i_shape = inputs.back(); - auto data_shape = inputs.front(); - auto r = data_shape.ndim(); - auto q = i_shape.ndim(); - - size_t k; - if(i_shape.dynamic()) - { - // the rank of the output is a function of k, so it must be fixed. - if(not i_shape.dyn_dims().back().is_fixed()) - { - MIGRAPHX_THROW( - "GATHERND: last dimension of indices tensor must be fixed (min=max)"); - } - k = i_shape.dyn_dims().back().min; - } - else - k = i_shape.lens().back(); - - // Begin input validation checks. - int output_ndim = int(q) + r - k - batch_dims - 1; - - if(k > r - batch_dims) - { - MIGRAPHX_THROW("GATHERND: Indices of length " + std::to_string(k) + - " cannot be used to access data of rank " + - std::to_string(r - batch_dims)); - } - - if(batch_dims >= q or batch_dims >= r) - { - MIGRAPHX_THROW("GATHERND: rank of an input cannot be less than batch_dims=" + - std::to_string(batch_dims)); - } - - if(output_ndim < 0) - { - MIGRAPHX_THROW("GATHERND: Indices too large for static data input: k=" + - std::to_string(k)); - } - - if(migraphx::none_of(inputs, [](auto v) { return v.dynamic(); })) - { - auto indices_lens_iter = i_shape.lens().begin(); - - // A rank 0 output is a scalar - if(output_ndim == 0) - return shape{data_shape.type(), {1}}; - - // Part of the output shape comes from indices tensor, part from data tensor - std::vector output_lens(output_ndim); - std::copy(indices_lens_iter, indices_lens_iter + (q - 1), output_lens.begin()); - // fill the rest of output shape from data tensor - if(k + batch_dims < r) - { - auto data_lens = data_shape.lens(); - std::copy(data_lens.begin() + batch_dims + k, - data_lens.end(), - output_lens.begin() + q - 1); - } - shape output_shape{data_shape.type(), output_lens}; - return output_shape; - } - else - { - // If one or both inputs are dynamic shapes, the output is dynamic. - // Make both inputs dynamic to simplify computations. - data_shape = data_shape.to_dynamic(); - i_shape = i_shape.to_dynamic(); - - // A rank 0 output is a scalar - if(output_ndim == 0) - return shape(data_shape.type(), {shape::dynamic_dimension({1, 1})}); - - // Part of the output shape comes from indices tensor, part from data tensor - std::vector output_dims(output_ndim); - std::copy(i_shape.dyn_dims().begin(), - i_shape.dyn_dims().begin() + q - 1, - output_dims.begin()); - - // fill the rest of output shape from data tensor - if(k + batch_dims < r) - { - auto data_dims = data_shape.dyn_dims(); - std::copy(data_dims.begin() + batch_dims + k, - data_dims.begin() + r, - output_dims.begin() + q - 1); - } - shape output_shape(data_shape.type(), output_dims); - return output_shape; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - visit_all(result, args[0])([&](auto output, auto data) { - args[1].visit([&](auto indices) { - auto indices_shape = indices.get_shape(); - auto indices_shape_lens = indices_shape.lens(); - auto data_shape = data.get_shape(); - auto data_shape_lens = data_shape.lens(); - auto k = indices_shape.lens().back(); - const auto num_slice_dims = k; - std::size_t num_slices = std::accumulate(indices_shape_lens.begin(), - indices_shape_lens.end() - 1, - 1, - std::multiplies()); - std::size_t slice_size = std::accumulate(data_shape_lens.begin() + k + batch_dims, - data_shape_lens.end(), - 1, - std::multiplies()); - std::size_t num_batches = std::accumulate(data_shape_lens.begin(), - data_shape_lens.begin() + batch_dims, - 1, - std::multiplies()); - std::size_t data_batch_stride = - std::accumulate(data_shape_lens.begin() + batch_dims, - data_shape_lens.end(), - 1, - std::multiplies()); - auto num_slices_per_batch = num_slices / num_batches; - - std::vector sizes_from_slice_dims(num_slice_dims); - { - auto running_product = slice_size; - for(std::size_t i = 0; i < num_slice_dims; ++i) - { - sizes_from_slice_dims[num_slice_dims - 1 - i] = running_product; - running_product *= data_shape_lens[batch_dims + num_slice_dims - 1 - i]; - } - } - - std::vector input_slice_offsets(num_slices); - par_for(num_slices, [&](const auto i) { - std::size_t batch_idx = i / num_slices_per_batch; - - auto slice_indices = indices.begin() + (i * num_slice_dims); - std::size_t relative_slice_offset = 0; - for(size_t dim_idx = 0; dim_idx < num_slice_dims; ++dim_idx) - { - int64_t index = *(slice_indices + dim_idx); - const std::size_t input_dim_idx = batch_dims + dim_idx; - const auto input_dim = data_shape_lens[input_dim_idx]; - if(index < -static_cast(input_dim) or - index >= static_cast(input_dim)) - MIGRAPHX_THROW("GatherND: index " + std::to_string(index) + - " is out of bounds for dim of len " + - std::to_string(input_dim)); - if(index < 0) - index += input_dim; - - relative_slice_offset += index * sizes_from_slice_dims[dim_idx]; - } - - input_slice_offsets[i] = - (batch_idx * data_batch_stride) + relative_slice_offset; - }); - - par_for(num_slices * slice_size, [&](const auto i) { - auto slice_offset = input_slice_offsets[i / slice_size]; - output[i] = data[slice_offset + i % slice_size]; - }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/get_tuple_elem.hpp b/docker/rocm/migraphx/include/migraphx/op/get_tuple_elem.hpp deleted file mode 100644 index e8e24d11c..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/get_tuple_elem.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_GET_TUPLE_ELEM_HPP -#define MIGRAPHX_GUARD_OPERATORS_GET_TUPLE_ELEM_HPP - -#include "migraphx/errors.hpp" -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct get_tuple_elem -{ - std::size_t index = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.index, "index")); - } - - std::string name() const { return "get_tuple_elem"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1).tuple_type(); - const auto& sub_shapes = inputs.at(0).sub_shapes(); - if(index >= sub_shapes.size()) - { - MIGRAPHX_THROW("GET_TUPLE_ELEM: index " + std::to_string(index) + " is out of range " + - std::to_string(sub_shapes.size())); - } - - return sub_shapes.at(index); - } - - argument compute(const shape&, std::vector args) const - { - assert(args.size() == 1); - auto vec_args = args.at(0).get_sub_objects(); - assert(index < vec_args.size()); - return vec_args.at(index); - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/greater.hpp b/docker/rocm/migraphx/include/migraphx/op/greater.hpp deleted file mode 100644 index 217a3d050..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/greater.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_GREATER_HPP -#define MIGRAPHX_GUARD_OPERATORS_GREATER_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct greater : binary -{ - std::string point_function() const { return ">"; } - auto apply() const - { - return [](auto x, auto y) { return x > y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/group_query_attention.hpp b/docker/rocm/migraphx/include/migraphx/op/group_query_attention.hpp deleted file mode 100644 index 60246f421..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/group_query_attention.hpp +++ /dev/null @@ -1,568 +0,0 @@ -#ifndef MIGRAPHX_GUARD_OPERATORS_GROUP_QUERY_ATTENTION_HPP -#define MIGRAPHX_GUARD_OPERATORS_GROUP_QUERY_ATTENTION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct gqa_parameters -{ - std::size_t batch_size = 0; // Batch size used by input - std::size_t sequence_length = 0; // Sequence length used by input - std::size_t hidden_size = 0; // Hidden size used by input - std::size_t head_size = 0; // Head size - std::size_t rotary_embedding_dim = 0; // Rotary embedding dimension. - std::size_t num_heads = 0; // num_heads = hidden_size / head_size - std::size_t max_sequence_length = 0; // Sequence length used by cos/sin cache - std::size_t head_stride = 0; // Head stride - std::size_t seq_stride = 0; // Sequence stride - std::size_t batch_stride = 0; // Batch stride - bool position_ids_use_batch = false; // Format of position ids - false is (1), true is - // (batch_size, sequence_length) - std::size_t seqlen_present_kv_cache = 0; // Sequence length of present kv-cache - // (4096 when using shared buffer) - bool past_present_share_buffer = false; // Whether to use same buffer for KV-cache - // inputs and outputs -}; - -struct group_query_attention -{ - bool do_rotary = false; - std::size_t kv_num_heads = 0; - int local_window_size = -1; - std::size_t num_heads = 1; - bool rotary_interleaved = false; - float scale = 1.0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.do_rotary, "do_rotary"), - f(self.kv_num_heads, "kv_num_heads"), - f(self.local_window_size, "local_window_size"), - f(self.num_heads, "num_heads"), - f(self.rotary_interleaved, "rotary_interleaved"), - f(self.scale, "scale")); - } - - std::string name() const { return "group_query_attention"; } - - shape compute_shape(std::vector inputs) const - { - auto query_lens = inputs.front().lens(); - std::size_t q_hidden_size = (query_lens[2] * num_heads) / (num_heads + 2 * kv_num_heads); - std::vector output_lens{query_lens.at(0), query_lens.at(1), q_hidden_size}; - shape output_shape{inputs.front().type(), output_lens}; - return shape({output_shape, inputs[3], inputs[4]}); - } - - template - void run_rotary_embedding(T input, - T cos_cache, - T sin_cache, - T output, - bool interleaved, - const std::size_t* pos_ids, - gqa_parameters parameters) const - { - const std::size_t batch_size = parameters.batch_size; - const std::size_t sequence_length = parameters.sequence_length; - const std::size_t n_heads = parameters.num_heads; - const std::size_t head_size = parameters.head_size; - const std::size_t head_stride = parameters.head_stride; - const std::size_t seq_stride = parameters.seq_stride; - const std::size_t batch_stride = parameters.batch_stride; - const std::size_t position_ids_use_batch = parameters.position_ids_use_batch; - const std::size_t rotary_emb_dim = parameters.rotary_embedding_dim; - const std::size_t half_rotary_emb_dim = rotary_emb_dim / 2; - - const std::size_t loop_len = batch_size * sequence_length * n_heads; - par_for(loop_len, [&](const auto idx) { - const std::size_t b = (idx / n_heads) / sequence_length; - const std::size_t s = (idx / n_heads) % sequence_length; - const std::size_t n = idx % n_heads; - const std::size_t block_offset = b * batch_stride + s * seq_stride + n * head_stride; - auto input_data = input + block_offset; - auto output_data = output + block_offset; - - // Cache is (M, H/2) or (M, rotary_embedding_dim/2) - const std::size_t position_id = - position_ids_use_batch ? pos_ids[b * sequence_length + s] : pos_ids[0] + s; - const std::size_t cache_offset = position_id * half_rotary_emb_dim; - auto cos_data = cos_cache + cache_offset; - auto sin_data = sin_cache + cache_offset; - - std::size_t cache_idx = 0; - float sign = 0.0; - std::size_t j = 0; - for(std::size_t i = 0; i < rotary_emb_dim; i++) - { - if(interleaved) - { - cache_idx = (i / 2) % half_rotary_emb_dim; - sign = (i % 2 == 0) ? -1.0 : 1.0; - j = (i % 2 == 0) ? i + 1 : i - 1; // i - sign - } - else - { - cache_idx = i % half_rotary_emb_dim; - sign = (i < half_rotary_emb_dim) ? -1.0 : 1.0; - j = (i + half_rotary_emb_dim) % rotary_emb_dim; - } - output_data[i] = input_data[i] * cos_data[cache_idx] + - sign * input_data[j] * sin_data[cache_idx]; - } - std::copy( - input_data + rotary_emb_dim, input_data + head_size, output_data + rotary_emb_dim); - }); - } - - template - void pack_v_into_rotary_qkv(gqa_parameters parameters, const T input, T output) const - { - const std::size_t loop_len = - parameters.batch_size * parameters.sequence_length * kv_num_heads; - par_for(loop_len, [&](const auto idx) { - const std::size_t b = (idx / kv_num_heads) / parameters.sequence_length; - const std::size_t s = (idx / kv_num_heads) % parameters.sequence_length; - const std::size_t n = idx % kv_num_heads; - const std::size_t block_offset = b * parameters.batch_stride + - s * parameters.seq_stride + n * parameters.head_stride; - const T input_data = input + block_offset; - T output_data = output + block_offset; - for(std::size_t i = 0; i < parameters.head_size; i++) - { - output_data[i] = input_data[i]; - } - }); - } - - template - void copy_data(T destination, const T source, std::size_t n) const - { - par_for(n, [&](auto i) { destination[i] = source[i]; }); - } - - template - T concat_state_chunk(const T past, - const T chunk, - T present, - std::size_t present_buff_chunk_length, - std::size_t past_buff_chunk_length, - std::size_t past_chunk_length, - std::size_t new_chunk_length, - bool is_prompt, - bool past_present_share_buffer, - std::ptrdiff_t i) const - { - T start = present + i * present_buff_chunk_length; - - T p = start; - if(not is_prompt) - { - if(not past_present_share_buffer) - { - const T src_past = past + i * past_buff_chunk_length; - copy_data(p, src_past, past_chunk_length); - } - p += past_chunk_length; - } - copy_data(p, chunk, new_chunk_length); - return start; - } - - template - void softmax_inplace(T score, std::size_t n, std::size_t d) const - { - par_for(n, [&](const auto j) { - auto x = score + j * d; - auto y = x; - - // e^x is represented as infinity if x is large enough, like 100.f. - // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if - // one or more item are large enough. a math transform as below is - // leveraged to get a stable softmax: e^xi/(e^x1 + ...e^xn) = e^(xi - - // max) / (e^(x1 - max) + ... + e^(xn - max)) - float max = -std::numeric_limits::infinity(); - for(std::size_t i = 0; i < d; i++) - { - if(max < x[i]) - max = x[i]; - } - for(std::size_t i = 0; i < d; i++) - { - y[i] = expf(x[i] - max); - } - - double sum = 0.0; - - for(std::size_t i = 0; i < d; i++) - { - sum += x[i]; - } - - for(std::size_t i = 0; i < d; i++) - { - y[i] = x[i] / static_cast(sum); - } - }); - } - - // Helper function to compute the attention probs. It does 2 things: - // attention_probs(B, N, S, T) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, T, H -> B, N, H, T) - // attention_probs(B, N, S, T) = Softmax(attention_probs) - template - void calculate_attention_probs(T attention_probs, // output buffer with size BxNxSxT - T query, // Q data. Its size is BxNxSxH - T key, // k data. Its size is BxNxLxH - U seqlens_k, // past sequence lengths tensor - T past_key, // past key only - T present_key, // present key only - shape::type_t dtype, - gqa_parameters params) const - { - const std::size_t batch_size = params.batch_size; - const std::size_t sequence_length = params.sequence_length; - const std::size_t head_size = params.head_size; - const std::size_t past_buffer_sequence_length = params.seqlen_present_kv_cache; - const std::size_t present_buffer_sequence_length = past_buffer_sequence_length; - const bool past_present_share_buffer = params.past_present_share_buffer; - - const bool is_prompt = sequence_length != 1; - const std::size_t packed_batch_stride = - (num_heads + 2 * kv_num_heads) * sequence_length * head_size; - const std::size_t kv_num_heads_factor = num_heads / kv_num_heads; - const std::size_t q_input_chunk_length = sequence_length * head_size; // S x H - const std::size_t kv_input_chunk_length = sequence_length * head_size; // L x H - const std::size_t past_buff_chunk_length = past_buffer_sequence_length * head_size; // L x H - const std::size_t present_buff_chunk_length = - present_buffer_sequence_length * head_size; // T x H - - const std::size_t loop_len = batch_size * num_heads; - const float alpha = scale == 0.0f ? 1.0f / std::sqrt(static_cast(head_size)) : scale; - - par_for(loop_len, [&](const auto i) { - const std::size_t batch_index = i / num_heads; - const std::size_t head_index = i % num_heads; - const std::size_t past_seqlen = - sequence_length == 1 ? seqlens_k[batch_index] : past_buffer_sequence_length; - const std::size_t past_chunk_length = past_seqlen * head_size; - const std::size_t total_seqlen = seqlens_k[batch_index] + 1; - - const std::size_t output_offset = i * sequence_length * present_buffer_sequence_length; - auto output = attention_probs + output_offset; - - auto k = key + packed_batch_stride * batch_index + - kv_input_chunk_length * (head_index / kv_num_heads_factor); - k = concat_state_chunk(past_key, - k, - present_key, - present_buff_chunk_length, - past_buff_chunk_length, - past_chunk_length, - kv_input_chunk_length, - is_prompt, - past_present_share_buffer, - i / kv_num_heads_factor); - - // Calculate Q*K' + AttentionMask - // original transposed each iteration - // A: Q (B x N x) S x H (B x N x) S x H S x H - // B: K' (B x N x) T x H (B x N x) H x T H x T - // C: attention_probs (B x N x) S x T (B x N x) S x T S x T - auto q = query + packed_batch_stride * batch_index + q_input_chunk_length * head_index; - - auto output_shape = - shape{dtype, {sequence_length, total_seqlen}, {present_buffer_sequence_length, 1}}; - auto q_shape = shape{dtype, {sequence_length, head_size}, {head_size, 1}}; - auto k_shape = shape{dtype, {head_size, total_seqlen}, {1, head_size}}; - auto cmat = make_view(output_shape, &(*output)); - auto amat = make_view(q_shape, &(*q)); - auto bmat = make_view(k_shape, &(*k)); - - gemm(cmat, amat, bmat, alpha, 0.0f); - - T output_softmax = output; - for(std::size_t seq = 0; seq < sequence_length; seq++) - { - std::size_t seq_causal_length = sequence_length == 1 ? total_seqlen : seq + 1; - if(local_window_size > 0 and seq_causal_length > local_window_size + 1) - { - for(std::size_t total_seq_id = 0; - total_seq_id < seq_causal_length - local_window_size - 1; - total_seq_id++) - { - output_softmax[total_seq_id] = 0.f; - } - softmax_inplace(output_softmax + seq_causal_length - local_window_size - 1, - 1, - local_window_size + 1); - } - else - { - softmax_inplace(output_softmax, 1, seq_causal_length); - } - // set causal [seq_causal_length, total_seqlen) to 0.f - for(std::size_t total_seq_id = seq_causal_length; total_seq_id < total_seqlen; - total_seq_id++) - { - output_softmax[total_seq_id] = 0.f; - } - - output_softmax += present_buffer_sequence_length; - } - }); - } - - template - void calculate_attention_score(T output, // buffer for the result with size BxSxNxH - const W attention_probs, // Attention probs with size BxNxSxT - const T val, // V value with size BxN_kvxSxH - const U seqlens_k, // past sequence lengths tensor - const T past_value, // past value only - T present_value, // present value only - shape::type_t dtype, - gqa_parameters params) const // whether Q, K, V are packed - { - const std::size_t batch_size = params.batch_size; - const std::size_t sequence_length = params.sequence_length; - const std::size_t head_size = params.head_size; - const std::size_t hidden_size = params.hidden_size; - const std::size_t past_buffer_sequence_length = params.seqlen_present_kv_cache; - const std::size_t present_buffer_sequence_length = past_buffer_sequence_length; - const bool past_present_share_buffer = params.past_present_share_buffer; - - const bool is_prompt = sequence_length != 1; - const std::size_t packed_batch_stride = - (num_heads + 2 * kv_num_heads) * sequence_length * head_size; - const std::size_t kv_num_heads_factor = num_heads / kv_num_heads; - const std::size_t kv_input_chunk_length = sequence_length * head_size; // L x H - const std::size_t past_buff_chunk_length = past_buffer_sequence_length * head_size; // L x H - const std::size_t present_buff_chunk_length = - present_buffer_sequence_length * head_size; // T x H - - auto loop_len = batch_size * num_heads; - par_for(loop_len, [&](const auto i) { - const std::size_t batch_index = i / num_heads; - const std::size_t head_index = i % num_heads; - const std::size_t past_seqlen = - sequence_length == 1 ? seqlens_k[batch_index] : past_buffer_sequence_length; - const std::size_t past_chunk_length = past_seqlen * head_size; - const std::size_t total_seqlen = seqlens_k[batch_index] + 1; - - auto v = val + packed_batch_stride * batch_index + - kv_input_chunk_length * (head_index / kv_num_heads_factor); - - v = concat_state_chunk(past_value, - v, - present_value, - present_buff_chunk_length, - past_buff_chunk_length, - past_chunk_length, - kv_input_chunk_length, - is_prompt, - past_present_share_buffer, - i / kv_num_heads_factor); - - T output_current = - output + (batch_index * sequence_length * num_heads + head_index) * head_size; - ptrdiff_t attention_probs_offset = sequence_length * present_buffer_sequence_length * i; - - auto output_shape = shape{dtype, {sequence_length, head_size}, {hidden_size, 1}}; - auto probs_shape = - shape{dtype, {sequence_length, total_seqlen}, {present_buffer_sequence_length, 1}}; - auto v_shape = shape{dtype, {total_seqlen, head_size}, {head_size, 1}}; - auto cmat = make_view(output_shape, &(*output_current)); - auto amat = make_view(probs_shape, &(*(attention_probs + attention_probs_offset))); - auto bmat = make_view(v_shape, &(*v)); - - gemm(cmat, amat, bmat, 1.0f, 0.0f); - }); - } - - template - void apply_attention(T qkv, - T past_key, - T past_value, - T output, - T present_key, - T present_value, - U seqlens_k, - T attention_probs, - gqa_parameters parameters, - shape::type_t dtype) const - { - const T k = qkv + num_heads * parameters.sequence_length * parameters.head_size; - calculate_attention_probs( - attention_probs, qkv, k, seqlens_k, past_key, present_key, dtype, parameters); - - const T v = - qkv + (num_heads + kv_num_heads) * parameters.sequence_length * parameters.head_size; - calculate_attention_score( - output, attention_probs, v, seqlens_k, past_value, present_value, dtype, parameters); - } - - argument compute(const shape& output_shape, std::vector args) const - { - auto q_shape = args[0].get_shape(); - auto q_lens = q_shape.lens(); - const std::size_t batch_size = q_lens[0]; - const std::size_t sequence_length = q_lens[1]; - auto past_key_shape = args[3].get_shape(); - auto past_key_lens = past_key_shape.lens(); - auto past_sequence_length = past_key_lens[2]; - std::size_t q_hidden_size = q_lens[2]; - std::size_t head_size = q_hidden_size / (num_heads + 2 * kv_num_heads); - q_hidden_size = head_size * num_heads; - std::size_t rotary_dim = args[7].get_shape().lens()[1] * 2; - - auto output_shape_0 = output_shape.sub_shapes().front(); - argument result{output_shape_0}; - argument qkv_rotary{ - shape{output_shape_0.type(), - {batch_size, num_heads + 2 * kv_num_heads, sequence_length, head_size}}}; - - shape kv_shape{output_shape_0.type(), - {batch_size, kv_num_heads, past_sequence_length, head_size}}; - argument present_k_out{kv_shape}; - argument present_v_out{kv_shape}; - argument attention_probs{shape{ - output_shape_0.type(), {batch_size, num_heads, sequence_length, past_sequence_length}}}; - - args[0] = args[0].reshape( - shape{output_shape_0.type(), - {batch_size, sequence_length, num_heads + 2 * kv_num_heads, head_size}}); - argument qkv{qkv_rotary.get_shape()}; - visit_all(qkv, args[0])([&](auto a, auto b) { - auto in_shape = args[0].get_shape(); - auto out_shape = qkv.get_shape(); - shape_for_each(in_shape, [&](const auto& idx) { - std::vector out_idx{idx[0], idx[2], idx[1], idx[3]}; - a(out_idx.begin(), out_idx.end()) = b(idx.begin(), idx.end()); - }); - }); - - visit_all(result, - qkv, - args[3], - args[4], - args[7], - args[8], - qkv_rotary, - present_k_out, - present_v_out, - attention_probs)([&](auto output, - auto query, - auto past_key, - auto past_value, - auto cos_cache, - auto sin_cache, - auto rotary_qkv, - auto present_k, - auto present_v, - auto attn_probs) { - visit_all(args[5])([&](auto seqlens_k) { - par_for(kv_shape.elements(), [&](auto i) { - present_k[i] = past_key[i]; - present_v[i] = past_value[i]; - }); - auto seq_stride = head_size; - auto head_stride = sequence_length * seq_stride; - auto batch_stride = num_heads + 2 * kv_num_heads; - auto position_ids_use_batch = sequence_length == 1; - std::vector pos_ids(sequence_length == 1 ? batch_size : 1); - if(sequence_length == 1) - { - std::copy(seqlens_k.begin(), seqlens_k.begin() + batch_size, pos_ids.begin()); - } - else - { - pos_ids[0] = 0; - } - auto q_input = query.begin(); - auto k_input = q_input + num_heads * sequence_length * head_size; - auto q_rotary = rotary_qkv.begin(); - auto k_rotary = q_rotary + num_heads * sequence_length * head_size; - - gqa_parameters gqa_params = {}; - gqa_params.batch_size = batch_size; - gqa_params.sequence_length = sequence_length; - gqa_params.hidden_size = q_hidden_size; - gqa_params.head_size = head_size; - gqa_params.rotary_embedding_dim = rotary_dim; - gqa_params.num_heads = num_heads; - gqa_params.max_sequence_length = sequence_length; - gqa_params.seq_stride = head_size; - gqa_params.head_stride = head_stride; - gqa_params.batch_stride = batch_stride; - gqa_params.position_ids_use_batch = position_ids_use_batch; - gqa_params.seqlen_present_kv_cache = past_sequence_length; - gqa_params.past_present_share_buffer = false; - - if(do_rotary) - { - run_rotary_embedding(q_input, - cos_cache.begin(), - sin_cache.begin(), - q_rotary, - rotary_interleaved, - pos_ids.data(), - gqa_params); - } - std::size_t kv_hidden_size = head_size * kv_num_heads; - gqa_params.num_heads = kv_num_heads; - gqa_params.hidden_size = kv_hidden_size; - - if(do_rotary) - { - run_rotary_embedding(k_input, - cos_cache.begin(), - sin_cache.begin(), - k_rotary, - rotary_interleaved, - pos_ids.data(), - gqa_params); - } - auto v_input = k_input + kv_num_heads * sequence_length * head_size; - auto v_rotary = k_rotary + kv_num_heads * sequence_length * head_size; - gqa_params.num_heads = num_heads; - - if(do_rotary) - { - pack_v_into_rotary_qkv(gqa_params, v_input, v_rotary); - } - else - { - rotary_qkv = query; - } - apply_attention(rotary_qkv.begin(), - past_key.begin(), - past_value.begin(), - output.begin(), - present_k.begin(), - present_v.begin(), - seqlens_k.begin(), - attn_probs.begin(), - gqa_params, - output_shape_0.type()); - }); - }); - - return {{result, present_k_out, present_v_out}}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/gru.hpp b/docker/rocm/migraphx/include/migraphx/op/gru.hpp deleted file mode 100644 index 186c2a053..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/gru.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_GRU_HPP -#define MIGRAPHX_GUARD_OPERATORS_GRU_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct gru -{ - std::size_t hidden_size = 1; - std::vector actv_funcs{sigmoid{}, tanh{}}; - rnn_direction direction = rnn_direction::forward; - float clip = 0.0f; - int linear_before_reset = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.hidden_size, "hidden_size"), - f(self.actv_funcs, "actv_func"), - f(self.direction, "direction"), - f(self.clip, "clip"), - f(self.linear_before_reset, "linear_before_reset")); - } - - std::string name() const { return "gru"; } - shape compute_shape(std::vector inputs) const - { - auto in_dims = inputs[0].lens(); - auto hidden_dims = inputs[2].lens(); - if(hidden_size != hidden_dims[2]) - { - MIGRAPHX_THROW("GRU: hidden size mismatch in attribute and input"); - } - - std::size_t num_directions = 1; - if(direction == rnn_direction::bidirectional) - { - num_directions = 2; - } - - if(num_directions != hidden_dims[0]) - { - MIGRAPHX_THROW("GRU: num_direction does not match the direction attribute"); - } - - std::vector out_dims(in_dims); - out_dims.insert(out_dims.begin() + 1, num_directions); - out_dims.back() = hidden_size; - - return {inputs[0].type(), out_dims}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/identity.hpp b/docker/rocm/migraphx/include/migraphx/op/identity.hpp deleted file mode 100644 index 9d80c5bfc..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/identity.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_IDENTITY_HPP -#define MIGRAPHX_GUARD_OPERATORS_IDENTITY_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct identity -{ - std::string name() const { return "identity"; } - shape compute_shape(std::vector inputs) const { return inputs.at(0); } - argument compute(shape, std::vector args) const { return args[0]; } - - value attributes() const { return {{"pointwise", true}, {"point_op", "${0}"}}; } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/if_op.hpp b/docker/rocm/migraphx/include/migraphx/op/if_op.hpp deleted file mode 100644 index 0b8ae78d8..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/if_op.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_IF_OP_HPP -#define MIGRAPHX_GUARD_OPERATORS_IF_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct if_op -{ - std::string name() const { return "if"; } - - shape compute_shape(const std::vector& inputs, std::vector mods) const - { - check_shapes{inputs, *this}.standard(); - if(mods.size() != 2) - { - MIGRAPHX_THROW("IF: operator should have two submodules."); - } - - auto out_shapes0 = mods[0]->get_output_shapes(); - auto out_shapes1 = mods[1]->get_output_shapes(); - if(not std::equal( - out_shapes1.begin(), out_shapes1.end(), out_shapes0.begin(), out_shapes0.end())) - { - MIGRAPHX_THROW("IF: output shapes of submodules must be the same."); - } - - return shape(out_shapes0); - } - - argument compute(const shape&, - const std::vector& args, - const std::vector& mods, - const std::function( - module_ref&, const std::unordered_map&)>& run) const - { - auto cond = args.front().at(); - module_ref mod = cond ? mods[0] : mods[1]; - std::unordered_map params; - - std::set pnames; - for(const_module_ref smod : mods) - { - auto names = smod->get_parameter_names(); - pnames.insert(names.begin(), names.end()); - } - - assert(pnames.size() < args.size()); - std::transform(pnames.begin(), - pnames.end(), - args.begin() + 1, - std::inserter(params, params.end()), - [](auto&& name, auto&& arg) { return std::make_pair(name, arg); }); - - auto results = run(mod, params); - return argument{results}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/im2col.hpp b/docker/rocm/migraphx/include/migraphx/op/im2col.hpp deleted file mode 100644 index 4912fb09e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/im2col.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_IM2COL_HPP -#define MIGRAPHX_GUARD_OPERATORS_IM2COL_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct im2col -{ - std::vector padding{0, 0}; - std::vector stride{1, 1}; - std::vector dilation{1, 1}; - - padding_mode_t padding_mode = default_; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.padding, "padding"), - f(self.stride, "stride"), - f(self.dilation, "dilation"), - f(self.padding_mode, "padding_mode")); - } - - std::string name() const { return "im2col"; } - - value attributes() const { return {{"normalize_padding", "padding"}}; } - - shape normalize_compute_shape(std::vector inputs) const - { - auto input = inputs[0]; - auto weights = inputs[1]; - auto batch_size = input.lens()[0]; - auto input_channels = weights.lens()[1]; - auto kernel_height = weights.lens()[2]; - auto kernel_width = weights.lens()[3]; - check_shapes{inputs, *this}.has(2); - if(batch_size != 1) - MIGRAPHX_THROW("im2col only support batch_size 1"); - - auto padding_h = 2 * padding[0]; - auto padding_w = 2 * padding[1]; - if(padding.size() == 2 * stride.size()) - { - padding_h = padding[0] + padding[2]; - padding_w = padding[1] + padding[3]; - } - auto output_height = std::size_t(std::max( - 1, - (input.lens()[2] - (1 + dilation[0] * (kernel_height - 1)) + padding_h) / stride[0] + - 1)); - auto output_width = std::size_t(std::max( - 1, - (input.lens()[3] - (1 + dilation[1] * (kernel_width - 1)) + padding_w) / stride[1] + - 1)); - - auto channels_col = kernel_height * kernel_width * input_channels; - return {input.type(), {output_height * output_width, channels_col}}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/isinf.hpp b/docker/rocm/migraphx/include/migraphx/op/isinf.hpp deleted file mode 100644 index 985644a3f..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/isinf.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ISINF_HPP -#define MIGRAPHX_GUARD_OPERATORS_ISINF_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct isinf : unary -{ - auto apply() const - { - return [&](auto x) { return std::isinf(static_cast(x)); }; - } - - std::string name() const { return "isinf"; } - - shape compute_shape(std::vector inputs) const - { - return unary::compute_shape(std::move(inputs)).with_type(shape::bool_type); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/isnan.hpp b/docker/rocm/migraphx/include/migraphx/op/isnan.hpp deleted file mode 100644 index a609a57ad..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/isnan.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ISNAN_HPP -#define MIGRAPHX_GUARD_OPERATORS_ISNAN_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct isnan : unary -{ - auto apply() const - { - return [](auto x) { return std::isnan(static_cast(x)); }; - } - - std::string name() const { return "isnan"; } - - shape compute_shape(std::vector inputs) const - { - return unary::compute_shape(std::move(inputs)).with_type(shape::bool_type); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/layout.hpp b/docker/rocm/migraphx/include/migraphx/op/layout.hpp deleted file mode 100644 index 19e83dc05..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/layout.hpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OP_LAYOUT_HPP -#define MIGRAPHX_GUARD_OP_LAYOUT_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct layout : unary -{ - std::vector permutation; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.permutation, "permutation")); - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1).only_dims(permutation.size()); - auto lens = inputs.at(0).lens(); - auto t = inputs.at(0).type(); - return shape::from_permutation(t, lens, permutation); - } - - auto apply() const - { - return [](auto x) { return x; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_OP_LAYOUT_HPP diff --git a/docker/rocm/migraphx/include/migraphx/op/leaky_relu.hpp b/docker/rocm/migraphx/include/migraphx/op/leaky_relu.hpp deleted file mode 100644 index 1575f3af5..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/leaky_relu.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LEAKY_RELU_HPP -#define MIGRAPHX_GUARD_OPERATORS_LEAKY_RELU_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct leaky_relu : unary -{ - float alpha = 0.01; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.alpha, "alpha")); - } - - std::string point_op() const { return "${function:where}(${0} > 0, ${0}, ${alpha} * ${0})"; } - - std::string name() const { return "leaky_relu"; } - - auto apply() const - { - return [&](auto x) { return x > 0 ? x : x * alpha; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/less.hpp b/docker/rocm/migraphx/include/migraphx/op/less.hpp deleted file mode 100644 index 9730d86d7..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/less.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LESS_HPP -#define MIGRAPHX_GUARD_OPERATORS_LESS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct less : binary -{ - std::string point_function() const { return "<"; } - auto apply() const - { - return [](auto x, auto y) { return x < y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/load.hpp b/docker/rocm/migraphx/include/migraphx/op/load.hpp deleted file mode 100644 index 0050e7150..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/load.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOAD_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOAD_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct load -{ - shape s; - std::size_t offset = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape"), f(self.offset, "offset")); - } - - std::string name() const { return "load"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(1); - return s; - } - argument compute(const shape&, const std::vector& args) const - { - if((offset + s.bytes()) > args[0].get_shape().bytes()) - MIGRAPHX_THROW("Load access is out of bounds"); - return argument{s, args[0].data() + offset}; - } - lifetime get_lifetime() const { return lifetime::borrow; } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } - - friend std::ostream& operator<<(std::ostream& os, const load& op) - { - os << op.name() << "["; - os << "offset=" << op.offset << ","; - os << "end=" << (op.offset + op.s.bytes()) << "]"; - return os; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/log.hpp b/docker/rocm/migraphx/include/migraphx/op/log.hpp deleted file mode 100644 index 12234d3d0..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/log.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOG_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOG_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct log : unary -{ - auto apply() const - { - return [](auto x) { return std::log(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/log2.hpp b/docker/rocm/migraphx/include/migraphx/op/log2.hpp deleted file mode 100644 index 5b993d530..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/log2.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOG2_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOG2_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct log2 : unary -{ - auto apply() const - { - - return [](auto x) { return std::log2(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/logical_and.hpp b/docker/rocm/migraphx/include/migraphx/op/logical_and.hpp deleted file mode 100644 index 09ecc174e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/logical_and.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOGICAL_AND_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOGICAL_AND_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct logical_and : binary -{ - std::string point_function() const { return "&&"; } - auto apply() const - { - return [](auto x, auto y) { return static_cast(x) and static_cast(y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/logical_or.hpp b/docker/rocm/migraphx/include/migraphx/op/logical_or.hpp deleted file mode 100644 index 10be93cdb..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/logical_or.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOGICAL_OR_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOGICAL_OR_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct logical_or : binary -{ - std::string point_function() const { return "||"; } - auto apply() const - { - return [](auto x, auto y) { return static_cast(x) or static_cast(y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/logical_xor.hpp b/docker/rocm/migraphx/include/migraphx/op/logical_xor.hpp deleted file mode 100644 index f2c822ad6..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/logical_xor.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOGICAL_XOR_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOGICAL_XOR_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct logical_xor : binary -{ - std::string point_function() const { return "^"; } - auto apply() const - { - return [](auto x, auto y) { return static_cast(x) xor static_cast(y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/logsoftmax.hpp b/docker/rocm/migraphx/include/migraphx/op/logsoftmax.hpp deleted file mode 100644 index 9df14eb49..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/logsoftmax.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOGSOFTMAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOGSOFTMAX_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct logsoftmax -{ - int64_t axis = 1; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "logsoftmax"; } - shape normalize_compute_shape(std::vector inputs) const - { - if(inputs.at(0).packed()) - { - return inputs.at(0); - } - else - { - auto lens = inputs.at(0).lens(); - return {inputs.at(0).type(), lens}; - } - } - - auto output() const - { - return [=](auto x, auto y) { return std::log(x / y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/loop.hpp b/docker/rocm/migraphx/include/migraphx/op/loop.hpp deleted file mode 100644 index a1b761814..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/loop.hpp +++ /dev/null @@ -1,171 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LOOP_HPP -#define MIGRAPHX_GUARD_OPERATORS_LOOP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct loop -{ - int64_t max_iterations = 10; - std::vector scan_output_directions = {}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.max_iterations, "max_iterations"), - f(self.scan_output_directions, "scan_output_directions")); - } - - std::string name() const { return "loop"; } - - shape compute_shape(const std::vector& inputs, std::vector mods) const - { - check_shapes{inputs, *this}.standard(); - if(mods.size() != 1) - { - MIGRAPHX_THROW("LOOP: operator should have one submodule."); - } - - const_module_ref mod = mods.front(); - auto mod_out_shapes = mod->get_output_shapes(); - auto dep_param_num = inputs.size() - 2; - - // first item of the mod output shapes is condition used in loop, - // which is not needed to compute output shape - mod_out_shapes.erase(mod_out_shapes.begin()); - std::vector ins_out_shapes(mod_out_shapes.begin(), - mod_out_shapes.begin() + dep_param_num); - mod_out_shapes.erase(mod_out_shapes.begin(), mod_out_shapes.begin() + dep_param_num); - for(const auto& out_s : mod_out_shapes) - { - auto lens = out_s.lens(); - lens.insert(lens.begin(), max_iterations); - ins_out_shapes.push_back({out_s.type(), lens}); - } - - return shape(ins_out_shapes); - } - - struct ref_loop - { - int64_t max_iterations = 0; - - template - void copy(context&, const argument& src, T& dst) const - { - dst = *src.cast(); - } - - template - void copy(context&, T src, const argument& dst) const - { - *dst.cast() = src; - } - - void append(const std::vector& iter_state, - const std::vector& concatenated_outputs, - const std::vector& scan_output_dirs, - int64_t curr_iter, - int64_t num_iters) const - { - assert(iter_state.size() == concatenated_outputs.size()); - for(auto i : range(iter_state.size())) - { - const auto& iter_stat = iter_state.at(i); - const auto& scan_out = concatenated_outputs.at(i); - - auto dir = scan_output_dirs.empty() ? 0 : scan_output_dirs[i]; - auto idx = (1 - dir) * curr_iter + dir * (num_iters - 1 - curr_iter); - - auto* in_data = iter_stat.data(); - auto* out_data = scan_out.data(); - std::size_t out_size = iter_stat.get_shape().bytes(); - assert((idx + 1) * out_size <= scan_out.get_shape().bytes()); - std::copy(in_data, in_data + out_size, out_data + idx * out_size); - } - } - - void set_zero(context&, const std::vector& concatenated_outputs, int iter) const - { - if(iter >= max_iterations) - return; - - for(const auto& out : concatenated_outputs) - { - auto s = out.get_shape(); - auto size = s.bytes() / max_iterations; - std::fill(out.data() + iter * size, out.data() + max_iterations * size, 0); - } - } - - std::unordered_map get_output_params(const module&) const { return {}; } - }; - - argument compute(context& ctx, - const shape& out_shape, - const std::vector& args, - const std::vector& mods, - const std::function( - module_ref&, const std::unordered_map&)>& run) const - { - // wrap up the arguments vector, so ref and gpu impl are the same - auto cpy_args = args; - bool in_cond = args.at(1).at(); - bool cond = in_cond; - int64_t iter = 0; - // insert iter and cond used in the loop - auto s_cond = args.at(1).get_shape(); - auto s_iter = args.at(0).get_shape(); - cpy_args.push_back({s_iter, &iter}); - cpy_args.push_back({s_cond, &cond}); - cpy_args.insert(cpy_args.end(), args.begin() + 2, args.end()); - - // add cond and mod outputs to the argument list - cpy_args.push_back(argument(s_cond)); - cpy_args.push_back(argument(out_shape)); - - // run loop - return run_loop(ref_loop{max_iterations}, scan_output_directions, ctx, cpy_args, mods, run); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/lrn.hpp b/docker/rocm/migraphx/include/migraphx/op/lrn.hpp deleted file mode 100644 index c4465d41d..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/lrn.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LRN_HPP -#define MIGRAPHX_GUARD_OPERATORS_LRN_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct lrn -{ - float alpha = 0.0001; - float beta = 0.75; - float bias = 1.0; - int size = 1; - std::string name() const { return "lrn"; } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.alpha, "alpha"), - f(self.beta, "beta"), - f(self.bias, "bias"), - f(self.size, "size")); - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1); - return inputs.front(); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/lstm.hpp b/docker/rocm/migraphx/include/migraphx/op/lstm.hpp deleted file mode 100644 index 04371e1c9..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/lstm.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_LSTM_HPP -#define MIGRAPHX_GUARD_OPERATORS_LSTM_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct lstm -{ - std::size_t hidden_size = 1; - std::vector actv_funcs{sigmoid{}, tanh{}, tanh{}}; - rnn_direction direction = rnn_direction::forward; - float clip = 0.0f; - int input_forget = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.hidden_size, "hidden_size"), - f(self.actv_funcs, "actv_func"), - f(self.direction, "direction"), - f(self.clip, "clip"), - f(self.input_forget, "input_forget")); - } - - std::string name() const { return "lstm"; } - shape compute_shape(std::vector inputs) const - { - auto in_dims = inputs[0].lens(); - auto hidden_dims = inputs[2].lens(); - if(hidden_size != hidden_dims[2]) - { - MIGRAPHX_THROW("LSTM: hidden size mismatch in attribute and input"); - } - - std::size_t num_directions = 1; - if(direction == rnn_direction::bidirectional) - { - num_directions = 2; - } - - if(num_directions != hidden_dims[0]) - { - MIGRAPHX_THROW("LSTM: num_direction does not match the direction attribute"); - } - - std::vector out_dims(in_dims); - out_dims.insert(out_dims.begin() + 1, num_directions); - out_dims.back() = hidden_size; - - return {inputs[0].type(), out_dims}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/max.hpp b/docker/rocm/migraphx/include/migraphx/op/max.hpp deleted file mode 100644 index f48531901..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/max.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_MAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_MAX_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct max : binary -{ - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = true; - return a; - } - auto apply() const - { - return [](auto x, auto y) { return std::max(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/min.hpp b/docker/rocm/migraphx/include/migraphx/op/min.hpp deleted file mode 100644 index 34c3f4eca..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/min.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_MIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_MIN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct min : binary -{ - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = true; - return a; - } - auto apply() const - { - return [](auto x, auto y) { return std::min(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/mod.hpp b/docker/rocm/migraphx/include/migraphx/op/mod.hpp deleted file mode 100644 index 38f947a35..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/mod.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_MOD_HPP -#define MIGRAPHX_GUARD_OPERATORS_MOD_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct mod : binary -{ - std::string name() const { return "mod"; } - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = false; - return a; - } - auto apply() const - { - return [](auto x, auto y) { return std::fmod((std::remainder(x, y)) + y, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/mul.hpp b/docker/rocm/migraphx/include/migraphx/op/mul.hpp deleted file mode 100644 index ab6b58fa7..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/mul.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_MUL_HPP -#define MIGRAPHX_GUARD_OPERATORS_MUL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct mul : binary -{ - value attributes() const - { - auto a = base_attributes(); - a["commutative"] = true; - return a; - } - std::string point_function() const { return "*"; } - auto apply() const - { - return [](auto x, auto y) { return x * y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/multibroadcast.hpp b/docker/rocm/migraphx/include/migraphx/op/multibroadcast.hpp deleted file mode 100644 index 80b728476..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/multibroadcast.hpp +++ /dev/null @@ -1,125 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_MULTIBROADCAST_HPP -#define MIGRAPHX_GUARD_OPERATORS_MULTIBROADCAST_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Broadcast multiple dimensions between two tensors. - * Two versions of this operator: 1 input and 2+ inputs. - * One input version uses output_lens attribute and broadcasts to it. - * 2+ inputs version broadcasts first input to the common shape at evaluation time. - */ -struct multibroadcast -{ - std::vector output_lens = {}; - - // optional attribute - std::vector output_dyn_dims = {}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.output_lens, "out_lens"), f(self.output_dyn_dims, "out_dyn_dims")); - } - - std::string name() const { return "multibroadcast"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has_at_least(1); - - auto t = inputs.at(0).type(); - auto s0 = inputs.at(0); - - if(s0.ndim() < 1) - { - MIGRAPHX_THROW("MULTIBROADCAST: input dimensions should be > 0"); - } - - if(inputs.size() == 1) - { - if(s0.dynamic()) - MIGRAPHX_THROW( - "MULTIBROADCAST: Single dynamic input shape not supported. Use two inputs."); - if(s0.ndim() > output_lens.size()) - { - MIGRAPHX_THROW("MULTIBROADCAST: input dimensions should <= output size"); - } - - auto offset = output_lens.size() - s0.ndim(); - for(std::ptrdiff_t i = s0.ndim() - 1; i >= 0; i--) - { - if(output_lens[i + offset] != s0.lens()[i] and s0.lens()[i] != 1) - { - MIGRAPHX_THROW("MULTIBROADCAST: input shape {" + to_string_range(s0.lens()) + - "} cannot be broadcasted to {" + to_string_range(output_lens) + - "}!"); - } - } - - return make_bcast_shape(s0, output_lens); - } - else - { - // 2+ inputs - if(std::any_of( - inputs.cbegin(), inputs.cend(), [](auto input) { return input.dynamic(); })) - { - if(not output_dyn_dims.empty()) - { - return {t, output_dyn_dims}; - } - return {t, compute_common_dyn_dims(inputs)}; - } - else - { - // output_lens will not be set for 2+ input version - auto bcast_lens = compute_common_lens(inputs); - return make_bcast_shape(s0, bcast_lens); - } - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/multinomial.hpp b/docker/rocm/migraphx/include/migraphx/op/multinomial.hpp deleted file mode 100644 index e48c6d319..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/multinomial.hpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/** - * * Multinomial or categorical distribution. Performs a sampling of random input - * and returns a count of - * each category, or bucket. This does not require the standard multinomial - * distribution but instead takes a probability distribution, i.e. cumulative - * distribution function (CDF) as its first input. - * - * Inputs: args[0] - a tensor of probabilities for each category. Values are - * cumulative density function - * totals as provided by operation prefix_scan_sum. Values are - * cumulative probabilities (i.e. start with any set of numbers > 0 - * and then apply prefix_scan_sum). Values do not need to be - * normalized to sum to 1; this is done in runtime computation. - * - * This input has Rank 2. Dimension 0 is batch #, so that there can be - * a different CDF for each iteration in the batch. The size of dimension - * 1 is the number of categories. - * - * args[1] - a tensor of random numbers. The last dimension is the sample - * size, i.e. the number of - * random samples in each iteration of the batch. Nominally - * has two dimensions where the first dimension is batch size, but - * any reshaping such that the total - * number of elements is (batch_size * sample_size) is legal. - * - * Values as created by a std::mt19937 like this: - * - * size_t sample_size = 100000; - * float seed = 0.0f; - * std::mt19937 gen(seed); - * std::uniform_real_distribution<> dis(0.0, 1.0); - * std::vector rand_samples(sample_size); - * std::generate(rand_samples.begin(), rand_samples.end(), [&]() { return - * dis(gen); }); - * - * Output: A 2D vector of category each input. Dimensions are (Input 1[first], Input - 2[last]). - * -*/ -#ifndef MIGRAPHX_GUARD_OPERATORS_MULTINOMIAL_HPP -#define MIGRAPHX_GUARD_OPERATORS_MULTINOMIAL_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct multinomial -{ - shape::type_t dtype = shape::type_t::int32_type; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.dtype, "dtype")); - } - - std::string name() const { return "multinomial"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2).only_dims(2); - - if(inputs.back().ndim() < 1) - MIGRAPHX_THROW("Multinomial: Second input shape (sample) has no dimensions"); - if(dtype == shape::bool_type) - MIGRAPHX_THROW("Multinomial: boolean output type invalid."); - - // Output takes one dimension from each of the two input shapes. If they are both fixed, - // return a static shape - if((not inputs.front().dynamic()) or (inputs.front().dyn_dims().front().is_fixed())) - { - if((not inputs.back().dynamic()) or (inputs.back().dyn_dims().back().is_fixed())) - { - size_t batch = {inputs.front().max_lens().front()}; - size_t sample_size{inputs.back().max_lens().back()}; - return {dtype, {batch, sample_size}}; - } - } - return {dtype, - {inputs.front().to_dynamic().dyn_dims().front(), - inputs.back().to_dynamic().dyn_dims().back()}}; - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - size_t batch_size = dyn_out.computed_shape.lens().front(); - size_t class_size = args[0].get_shape().lens().back(); - size_t sample_size = dyn_out.computed_shape.lens().back(); - - visit_all(args[0], args[1])([&](auto cdf, auto dist) { - result.visit([&](auto output) { - par_for(batch_size * sample_size, [&](auto i) { - auto idx = args[1].get_shape().multi(i); - auto cdf_begin = cdf.begin() + (idx[0] * class_size); - auto cdf_end = cdf_begin + class_size; - - // std::upper_bound returns an iterator to the bucket the value belongs in, - // when normalized by the probability distribution dist - auto sample_iter = - std::upper_bound(cdf_begin, cdf_end, dist[i] * *(std::prev(cdf_end))); - // convert iterator to an integer index - output[i] = std::distance(cdf_begin, sample_iter); - }); - }); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/name.hpp b/docker/rocm/migraphx/include/migraphx/op/name.hpp deleted file mode 100644 index 00d7bc360..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/name.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_NAME_HPP -#define MIGRAPHX_GUARD_RTGLIB_NAME_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/// Create name from class -template -struct op_name -{ - std::string name() const - { - static const std::string& name = get_type_name(); - return name.substr(name.rfind("::") + 2); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/nearbyint.hpp b/docker/rocm/migraphx/include/migraphx/op/nearbyint.hpp deleted file mode 100644 index ce1917750..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/nearbyint.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_NEARBYINT_HPP -#define MIGRAPHX_GUARD_OPERATORS_NEARBYINT_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { -struct nearbyint : unary -{ - auto apply() const - { - return [](auto x) { - auto rounding_mode = fegetround(); - fesetround(FE_TONEAREST); - auto result = std::nearbyint(x); - fesetround(rounding_mode); - return result; - }; - } -}; -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/neg.hpp b/docker/rocm/migraphx/include/migraphx/op/neg.hpp deleted file mode 100644 index 3c677ba5e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/neg.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_NEG_HPP -#define MIGRAPHX_GUARD_OPERATORS_NEG_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct neg : unary -{ - std::string point_function() const { return "-"; } - auto apply() const - { - return [](auto x) { return -x; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/nonmaxsuppression.hpp b/docker/rocm/migraphx/include/migraphx/op/nonmaxsuppression.hpp deleted file mode 100644 index d212dcbba..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/nonmaxsuppression.hpp +++ /dev/null @@ -1,412 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_NONMAXSUPPRESSION_HPP -#define MIGRAPHX_GUARD_OPERATORS_NONMAXSUPPRESSION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* -https://github.com/onnx/onnx/blob/main/docs/Operators.md#NonMaxSuppression - -Filter out boxes that have high intersection-over-union (IOU) overlap with previously selected -boxes. Bounding boxes with score less than score_threshold are removed. Bounding box format is -indicated by attribute center_point_box. Note that this algorithm is agnostic to where the origin is -in the coordinate system and more generally is invariant to orthogonal transformations and -translations of the coordinate system; thus translating or reflections of the coordinate system -result in the same boxes being selected by the algorithm. The selected_indices output is a set of -integers indexing into the input collection of bounding boxes representing the selected boxes. The -bounding box coordinates corresponding to the selected indices can then be obtained using the Gather -or GatherND operation. - -Version -This version of the operator has been available since version 11 of the default ONNX operator set. -Other versions of this operator: 10 - -Attributes -center_point_box : int (default is 0) -Integer indicate the format of the box data. The default is 0. 0 - the box data is supplied as [y1, -x1, y2, x2] where (y1, x1) and (y2, x2) are the coordinates of any diagonal pair of box corners and -the coordinates can be provided as normalized (i.e., lying in the interval [0, 1]) or absolute. -Mostly used for TF models. 1 - the box data is supplied as [x_center, y_center, width, height]. -Mostly used for Pytorch models. - -Inputs (2 - 5) ---------------------------------------------------------------------------------------------------------------------- -boxes : tensor(float) -An input tensor with shape [num_batches, spatial_dimension, 4]. -The single box data format is indicated by center_point_box. - -scores : tensor(float) -An input tensor with shape [num_batches, num_classes, spatial_dimension] - -max_output_boxes_per_class (optional) : tensor(int64) -Integer representing the maximum number of boxes to be selected per batch per class. -It is a scalar. Default to 0, which means no output. - -iou_threshold (optional) : tensor(float) -Float representing the threshold for deciding whether boxes overlap too much with respect to IOU. -It is scalar. Value range [0, 1]. Default to 0. - -score_threshold (optional) : tensor(flo187Gat) -Float representing the threshold for deciding when to remove boxes based on score. It is a scalar. ----------------------------------------------------------------------------------------------------------------------- -Outputs -selected_indices : tensor(int64) -selected indices from the boxes tensor. [num_selected_indices, 3], -the selected index format is [batch_index, class_index, box_index]. ----------------------------------------------------------------------------------------------------------------------- -*/ -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct nonmaxsuppression -{ - bool center_point_box = false; - bool use_dyn_output = false; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.center_point_box, "center_point_box"), - f(self.use_dyn_output, "use_dyn_output")); - } - - std::string name() const { return "nonmaxsuppression"; } - - shape compute_shape(std::vector inputs) const - { - // requires at least 2 inputs - check_shapes{{inputs.at(0), inputs.at(1)}, *this, true}.only_dims(3).same_ndims(); - auto boxes_max_lens = inputs.at(0).max_lens(); - // num batches * num boxes - const auto max_num_boxes = boxes_max_lens.at(0) * boxes_max_lens.at(1); - - auto fixed_shape_error_check = [&]() { - auto lens = inputs.front().lens(); - if(lens[1] != inputs.at(1).lens()[2]) - { - MIGRAPHX_THROW( - "NonMaxSuppression: spatial dimension mismatch between boxes and scores input"); - } - if(lens[0] != inputs.at(1).lens()[0]) - { - MIGRAPHX_THROW( - "NonMaxSuppression: number of batches mismatch between boxes and scores input"); - } - }; - - if(use_dyn_output) - { - if(inputs.at(0).dynamic()) - { - // both boxes and scores should be dynamic - // check dynamic dimensions are consistent - const auto boxes_dims = inputs.at(0).dyn_dims(); - const auto scores_dims = inputs.at(1).dyn_dims(); - if(boxes_dims.at(1) != scores_dims.at(2)) - { - MIGRAPHX_THROW("NonMaxSuppression: dynamic spatial dimension mismatch between " - "boxes and scores input"); - } - if(boxes_dims.at(0) != scores_dims.at(0)) - { - MIGRAPHX_THROW("NonMaxSuppression: dynamic number of batches mismatch between " - "boxes and scores input"); - } - } - else if(inputs.at(1).dynamic()) - { - // scores has dynamic shape, boxes fixed shape - // check that it is only a dynamic number of classes - const auto scores_dims = inputs.at(1).dyn_dims(); - const auto boxes_lens = inputs.at(0).lens(); - if(not scores_dims.at(0).is_fixed() or scores_dims.at(0).max != boxes_lens.at(0)) - { - MIGRAPHX_THROW("NonMaxSuppression: scores dynamic num_classes; num_batches not " - "fixed or mismatched"); - } - if(not scores_dims.at(2).is_fixed() or scores_dims.at(2).max != boxes_lens.at(1)) - { - MIGRAPHX_THROW("NonMaxSuppression: scores dynamic num_classes; " - "spatial_dimension not fixed or mismatches"); - } - } - else - { - fixed_shape_error_check(); - } - std::vector out_lens = {}; - out_lens.push_back({0, max_num_boxes}); - out_lens.push_back({3, 3}); - return {shape::int64_type, out_lens}; - } - else - { - if(inputs.at(0).dynamic() or inputs.at(1).dynamic()) - { - MIGRAPHX_THROW( - "NonMaxSuppression: dynamic input shape with use_dyn_output set to false"); - } - fixed_shape_error_check(); - std::vector out_lens = {max_num_boxes, 3}; - return {shape::int64_type, out_lens}; - } - } - - struct box - { - std::array x; - std::array y; - - void sort() - { - if(x[0] > x[1]) - { - std::swap(x[0], x[1]); - } - if(y[0] > y[1]) - { - std::swap(y[0], y[1]); - } - } - - std::array& operator[](std::size_t i) { return i == 0 ? x : y; } - - double area() const - { - assert(x[0] <= x[1]); - assert(y[0] <= y[1]); - return (x[1] - x[0]) * (y[1] - y[0]); - } - }; - - template - box batch_box(T boxes, std::size_t box_idx) const - { - box result{}; - auto start = boxes + 4 * box_idx; - if(center_point_box) - { - double half_width = start[2] / 2.0; - double half_height = start[3] / 2.0; - double x_center = start[0]; - double y_center = start[1]; - result.x = {x_center - half_width, x_center + half_width}; - result.y = {y_center - half_height, y_center + half_height}; - } - else - { - result.x = {static_cast(start[1]), static_cast(start[3])}; - result.y = {static_cast(start[0]), static_cast(start[2])}; - } - - result.sort(); - - return result; - } - - inline bool suppress_by_iou(box b1, box b2, double iou_threshold) const - { - const double area1 = b1.area(); - const double area2 = b2.area(); - - if(area1 <= .0f or area2 <= .0f) - { - return false; - } - - box intersection{}; - for(auto i : range(2)) - { - intersection[i][0] = std::max(b1[i][0], b2[i][0]); - intersection[i][1] = std::min(b1[i][1], b2[i][1]); - if(intersection[i][0] > intersection[i][1]) - { - return false; - } - } - - const double intersection_area = intersection.area(); - const double union_area = area1 + area2 - intersection_area; - - if(union_area <= .0f) - { - return false; - } - - const double intersection_over_union = intersection_area / union_area; - - return intersection_over_union > iou_threshold; - } - - // filter boxes below score_threshold - template - std::vector> - filter_boxes_by_score(T scores_start, std::size_t num_boxes, double score_threshold) const - { - std::vector> boxes_heap; - int64_t box_idx = 0; - - if(score_threshold > 0.0) - { - transform_if( - scores_start, - scores_start + num_boxes, - std::back_inserter(boxes_heap), - [&](auto sc) { - box_idx++; - return sc >= score_threshold; - }, - [&](auto sc) { return std::make_pair(sc, box_idx - 1); }); - } - else - { // score is irrelevant, just push into boxes_heap and make a score-index pair - std::transform(scores_start, - scores_start + num_boxes, - std::back_inserter(boxes_heap), - [&](auto sc) { - box_idx++; - return std::make_pair(sc, box_idx - 1); - }); - } - par_sort(boxes_heap.begin(), boxes_heap.end(), std::greater>{}); - return boxes_heap; - } - - template - std::size_t compute_nms(Output output, - Boxes boxes, - Scores scores, - std::size_t max_output_boxes_per_class, - double iou_threshold, - double score_threshold) const - { - std::fill(output.begin(), output.end(), 0); - const auto& lens = scores.get_shape().lens(); - const auto num_batches = lens[0]; - const auto num_classes = lens[1]; - const auto num_boxes = lens[2]; - // boxes of a class with NMS applied [score, index] - std::vector selected_indices; - // iterate over batches and classes - shape comp_s{shape::double_type, {num_batches, num_classes}}; - shape_for_each(comp_s, [&](const auto& idx) { - auto batch_idx = idx[0]; - auto class_idx = idx[1]; - // index offset for this class - auto scores_start = scores.begin() + (batch_idx * num_classes + class_idx) * num_boxes; - // iterator to first value of this batch - auto batch_boxes_start = boxes.begin() + batch_idx * num_boxes * 4; - auto boxes_heap = filter_boxes_by_score(scores_start, num_boxes, score_threshold); - int64_t selected_boxes_inside_class = 0; - while(not boxes_heap.empty() and - selected_boxes_inside_class < max_output_boxes_per_class) - { - // select next top scorer box and remove any boxes from boxes_heap that exceeds IOU - // threshold with the selected box - const auto next_top_score = boxes_heap.front(); - auto next_box = batch_box(batch_boxes_start, next_top_score.second); - auto next_box_idx = next_top_score.second; - - selected_boxes_inside_class++; - selected_indices.push_back(batch_idx); - selected_indices.push_back(class_idx); - selected_indices.push_back(next_box_idx); - - std::vector> remainder_boxes(boxes_heap.size()); - - auto it = par_copy_if( - boxes_heap.begin() + 1, - boxes_heap.end(), - remainder_boxes.begin(), - [&](auto iou_candidate_box) { - auto iou_box = batch_box(batch_boxes_start, iou_candidate_box.second); - return not this->suppress_by_iou(iou_box, next_box, iou_threshold); - }); - - remainder_boxes.resize(it - remainder_boxes.begin()); - boxes_heap = remainder_boxes; - } - }); - std::copy(selected_indices.begin(), selected_indices.end(), output.begin()); - return selected_indices.size() / 3; - } - - argument compute(const shape& output_shape, std::vector args) const - { - // make buffer of maximum size - shape max_output_shape = {output_shape.type(), output_shape.max_lens()}; - argument result{max_output_shape}; - - std::size_t max_output_boxes_per_class = - (args.size() > 2) ? (args.at(2).at()) : 0; - if(max_output_boxes_per_class == 0) - { - return result; - } - double iou_threshold = (args.size() > 3) ? (args.at(3).at()) : 0.0f; - double score_threshold = (args.size() > 4) ? (args.at(4).at()) : 0.0f; - std::size_t num_selected = 0; - - result.visit([&](auto output) { - visit_all(args[0], args[1])([&](auto boxes, auto scores) { - num_selected = compute_nms(output, - boxes, - scores, - max_output_boxes_per_class, - iou_threshold, - score_threshold); - }); - }); - if(use_dyn_output) - { - return result.reshape({output_shape.type(), {num_selected, 3}}); - } - else - { - return result; - } - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/nonzero.hpp b/docker/rocm/migraphx/include/migraphx/op/nonzero.hpp deleted file mode 100644 index e14d62a05..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/nonzero.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_NONZERO_HPP -#define MIGRAPHX_GUARD_OPERATORS_NONZERO_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct nonzero -{ - std::string name() const { return "nonzero"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1).standard(); - auto elem_num = inputs[0].elements(); - auto dim_num = inputs[0].lens().size(); - std::vector out_lens = {dim_num, elem_num}; - - return {shape::int64_type, out_lens}; - } - - argument compute(const shape& output_shape, std::vector args) const - { - std::vector> vec_idx; - auto s = args.front().get_shape(); - args.front().visit([&](auto v) { - shape_for_each(s, [&](const auto& idx_v, size_t idx) { - if(not float_equal(v[idx], 0)) - { - vec_idx.push_back(idx_v); - } - }); - }); - - argument result{output_shape}; - result.visit([&](auto output) { - std::fill(output.begin(), output.end(), 0); - par_for(vec_idx.size(), [&](auto i) { - for(std::size_t j = 0; j < vec_idx.front().size(); ++j) - { - output[output_shape.index({j, i})] = vec_idx[i][j]; - } - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/normalize_attribute.hpp b/docker/rocm/migraphx/include/migraphx/op/normalize_attribute.hpp deleted file mode 100644 index c545c2045..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/normalize_attribute.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_OP_NORMALIZE_ATTRIBUTE_HPP -#define MIGRAPHX_GUARD_OPERATORS_OP_NORMALIZE_ATTRIBUTE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * `normalize_attribute` settings: - * Note that default options are not included as enums. - * 1. `use_input` (default) vs. `use_output`: - * Affects the rank of the attribute. - * `use_input -> lens.size()`, `use_output -> lens.size() + vec.size()`. - * 2. use_rank (default) vs use_len: - * `use_rank` sets the max value/index of the attribute as the rank of lens. - * `use_lens` sets the max value/index as the corresponding value in lens at the axes index. - * Uses the dynamic_dimension.max value for dynamic shapes. Returns the original vector - * (no normalization) if any of dynamic_dimension[axes] are not fixed. - * 3. `clip_min` vs. `not_clip_min` (default): - * Clip values less than the minimum to the minimum or not. - * 4. `include_min` vs. `exclude_min` (default): - * Include or exclude the minimum value/index for range checking and clipping. - * 5. `clip_max` vs. `not_clip_max` (default): - * Clip values greater than the maximum or not. - * 6. `include_max` vs. `exclude_max` (default): - * Include or exclude the maximum value/index for range checking and clipping. - * 7. `normalize_padding`: - * To normalize the padding to `2*(pad ndim)` dimensions. - */ -enum class normalize_attribute -{ - use_output, - use_len, - clip_max, - clip_min, - include_max, - include_min, - normalize_padding -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/onehot.hpp b/docker/rocm/migraphx/include/migraphx/op/onehot.hpp deleted file mode 100644 index 205df03fa..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/onehot.hpp +++ /dev/null @@ -1,172 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ONEHOT_HPP -#define MIGRAPHX_GUARD_OPERATORS_ONEHOT_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Produces a one-hot tensor. - * Called with `axis` attribute that defaults to the last output axis - * Constant depth: `onehot(indices, values), depth attribute must be set; - * Variable depth: `onehot(indices, depth, values)`; - * `indicies` as a N rank tensor of indices where value is `on_value` - * `depth` scalar with the number of classes for the one-hot dimension - * `values` `[off_value, on_value]` - * `axis` which axis to add the one-hot dimension to - * For axis = 0 and rank(indices) = 2: - * output is A[indicies[j, k], j, k] = on_value; A[i, j, k] = off_value otherwise - * Can be simplified to other operators when `indices` has a static shape and - * `depth` is constant at compile-time. - */ -struct onehot -{ - // cannot use normalize_attribute here since rank(output_shape) = rank(indices) + 1 - int64_t axis = -1; - // optional depth attribute for static output shape - std::optional depth; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.depth, "depth")); - } - - std::string name() const { return "onehot"; } - - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this, true}.has(2, 3); - const auto& indices_shape = inputs[0]; - auto normalized_axis = - (this->axis < 0) ? this->axis + indices_shape.ndim() + 1 : this->axis; - if(normalized_axis > indices_shape.ndim()) - { - MIGRAPHX_THROW("ONEHOT: axis is out of range"); - } - if(depth.has_value()) - { - if(depth.value() < 0) - { - MIGRAPHX_THROW("ONEHOT: negative depth attribute value"); - } - check_shapes{inputs, *this, true}.has(2); - // `values` should have static shape - (void)check_shapes{inputs.begin() + 1, inputs.end(), *this, false}; - const auto& values_shape = inputs[1]; - if(not indices_shape.dynamic()) - { - // static output shape - auto output_lens = indices_shape.lens(); - output_lens.insert(output_lens.begin() + normalized_axis, depth.value()); - return {values_shape.type(), output_lens}; - } - // dynamic output shape - auto output_dds = indices_shape.to_dynamic().dyn_dims(); - std::size_t depth_val = depth.value(); - output_dds.insert(output_dds.begin() + normalized_axis, - shape::dynamic_dimension{depth_val, depth_val}); - return {values_shape.type(), output_dds}; - } - else - { - // dynamic output shape - check_shapes{inputs, *this, true}.has(3); - // `depth` and `values` should have static shape - (void)check_shapes{inputs.begin() + 1, inputs.end(), *this, false}; - const auto& values_shape = inputs[2]; - auto output_dds = indices_shape.to_dynamic().dyn_dims(); - std::size_t max_val = std::numeric_limits::max(); - output_dds.insert(output_dds.begin() + normalized_axis, - shape::dynamic_dimension{0, max_val}); - return {values_shape.type(), output_dds}; - } - } - - argument compute(const shape&, std::vector args) const - { - auto indices_shape = args[0].get_shape(); - int64_t depth_val; - auto values_iter = args.begin(); - if(this->depth.has_value()) - { - assert(args.size() == 2); - depth_val = depth.value(); - values_iter += 1; - } - else - { - assert(args.size() == 3); - args[1].visit([&](auto d) { depth_val = d(0); }); - values_iter += 2; - } - if(depth_val < 0) - { - MIGRAPHX_THROW("ONEHOT: negative depth value"); - } - auto output_lens = indices_shape.lens(); - auto normalized_axis = (axis < 0) ? axis + indices_shape.ndim() + 1 : axis; - output_lens.insert(output_lens.begin() + normalized_axis, depth_val); - shape output_shape{values_iter->get_shape().type(), output_lens}; - - argument result{output_shape}; - visit_all(result, *values_iter)([&](auto output, auto values) { - auto off_value = values(0); - auto on_value = values(1); - // fill result with off_value - par_for(output_shape.elements(), [&](auto i) { output[i] = off_value; }); - args[0].visit([&](auto indices) { - auto ind_s = indices.get_shape(); - shape_for_each(ind_s, [&](const auto& idx) { - auto index = indices(idx.begin(), idx.end()); - // normalize negative indices - index = (index < 0) ? index + depth_val : index; - // no on_value if index is out of range - if(index >= 0 and index < depth_val) - { - std::vector out_idx = idx; - out_idx.insert(out_idx.begin() + normalized_axis, index); - output(out_idx.begin(), out_idx.end()) = on_value; - } - }); - }); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/outline.hpp b/docker/rocm/migraphx/include/migraphx/op/outline.hpp deleted file mode 100644 index 0c5d5878e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/outline.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_OUTLINE_HPP -#define MIGRAPHX_GUARD_OPERATORS_OUTLINE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct outline -{ - shape s; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape")); - } - - std::string name() const { return "outline"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return s; - } - argument compute(const shape&, const std::vector&) const { return {s, nullptr}; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/pack_int4.hpp b/docker/rocm/migraphx/include/migraphx/op/pack_int4.hpp deleted file mode 100644 index 69d8ef71a..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/pack_int4.hpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_PACK_INT4_HPP -#define MIGRAPHX_GUARD_OPERATORS_PACK_INT4_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { -struct pack_int4 -{ - int64_t axis = -1; - - std::string name() const { return "pack_int4"; } - - value attributes() const - { - value normalize = value::object{}; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - migraphx::shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.same_dims().has(1); - auto in_shape = inputs.front(); - if(in_shape.type() != migraphx::shape::int8_type and - in_shape.type() != migraphx::shape::uint8_type) - { - MIGRAPHX_THROW("PACK_INT4: Only Int8 or Uint8 is supported for packing"); - } - auto new_lens = in_shape.lens(); - if(new_lens[axis] % 2 != 0) - { - MIGRAPHX_THROW("PACK_INT4: Can not pack axis that has odd lengths"); - } - new_lens[axis] /= 2; - return {in_shape.type(), new_lens}; - } - - argument compute(const shape& output_shape, std::vector args) const - { - auto input = args.front(); - auto in_shape = input.get_shape(); - - argument result{output_shape}; - - visit_all(result, input)([&](auto out, auto inp) { - par_for(output_shape.elements(), [&](auto i) { - using type = typename decltype(inp)::value_type; - type min_4bit; // clip min value - type max_4bit; // clip max value - - if constexpr(std::is_signed{}) - { - min_4bit = -8; - max_4bit = 7; - } - else - { - min_4bit = 0; - max_4bit = 15; - } - - auto data_idx = output_shape.multi(i); - auto in_data_multi_idx = data_idx; - in_data_multi_idx[axis] *= 2; - type val1 = inp[in_data_multi_idx]; - in_data_multi_idx[axis] += 1; - type val2 = inp[in_data_multi_idx]; - - // clip: - val1 = std::min(std::max(val1, min_4bit), max_4bit); - val2 = std::min(std::max(val2, min_4bit), max_4bit); - - // pack: - // the bit operations are forced into uint8_t mode, - // and this would avoid compiler warnings as well. - uint8_t val_ui8_1 = static_cast(val1); - uint8_t val_ui8_2 = static_cast(val2); - out[i] = (val_ui8_2 << 4) | (val_ui8_1 & 0xf); // NOLINT(hicpp-signed-bitwise) - }); - }); - return result; - } -}; -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/pad.hpp b/docker/rocm/migraphx/include/migraphx/op/pad.hpp deleted file mode 100644 index 478299624..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/pad.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_PAD_HPP -#define MIGRAPHX_GUARD_OPERATORS_PAD_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct pad -{ - std::vector pads; - float value = 0.0f; - enum pad_op_mode_t - { - constant_pad, - reflect_pad, - edge_pad - }; - pad_op_mode_t mode = constant_pad; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.mode, "mode"), f(self.pads, "pads"), f(self.value, "value")); - } - - std::string name() const { return "pad"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - const auto& s0 = inputs.front(); - if(s0.dynamic()) - { - auto out_dyn_dims = s0.dyn_dims(); - for(std::size_t i = 0; i < s0.ndim(); ++i) - { - out_dyn_dims[i] += pads[i] + pads[i + s0.ndim()]; - } - return {s0.type(), out_dyn_dims}; - } - else - { - auto&& idims = s0.lens(); - std::vector rdims(idims.begin(), idims.end()); - std::size_t num_dims = rdims.size(); - for(std::size_t i = 0; i < num_dims; i++) - { - rdims[i] += pads[i] + pads[i + num_dims]; - } - return s0.with_lens(rdims); - } - } - - std::size_t pad_ndims() const - { - assert(pads.size() % 2 == 0); - return pads.size() / 2; - } - - bool symmetric() const - { - std::size_t num_dims = pads.size() / 2; - return std::equal( - pads.begin(), pads.begin() + num_dims, pads.begin() + num_dims, pads.end()); - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/pointwise.hpp b/docker/rocm/migraphx/include/migraphx/op/pointwise.hpp deleted file mode 100644 index c76276a9f..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/pointwise.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OP_POINTWISE_HPP -#define MIGRAPHX_GUARD_OP_POINTWISE_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct pointwise -{ - std::string name() const { return "pointwise"; } - - shape compute_shape(const std::vector& inputs, std::vector mods) const - { - if(mods.size() != 1) - { - MIGRAPHX_THROW("should have one submodule."); - } - if(inputs.empty()) - MIGRAPHX_THROW("pointwise should have at least one input"); - auto* pm = mods.front(); - auto pnames = pm->get_parameter_names(); - check_shapes{inputs, *this}.has(pnames.size()).same_dims(); - - auto result = pm->compute_shapes( - inputs, - {.name = name(), .strict_type = true, .scalar_const_out_lens = inputs.front().lens()}); - if(result.size() == 1) - return result.front(); - return shape{result}; - } - - argument compute(const shape& output_shape, - const std::vector& args, - const std::vector& mods, - const std::function( - module_ref&, const std::unordered_map&)>& run) const - { - argument output{output_shape}; - auto* pm = mods.front(); - auto pnames = pm->get_parameter_names(); - std::sort(pnames.begin(), pnames.end()); - - par_for(args[0].get_shape().elements(), [&](auto i) { - std::unordered_map params; - - std::transform( - pnames.begin(), - pnames.end(), - args.begin(), - std::inserter(params, params.end()), - [&](auto&& name, auto&& arg) { return std::make_pair(name, arg.element(i)); }); - - auto results = run(pm, params); - assert(results.size() == output.get_sub_objects().size() or - (results.size() == 1 and output.get_sub_objects().empty())); - std::vector outputs; - if(results.size() == 1) - outputs = {output.share()}; - else - outputs = output.share().get_sub_objects(); - for(auto j : range(results.size())) - visit_all(outputs[j], results[j])([&](auto out, auto x) { out[i] = x.front(); }); - }); - return output; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_OP_POINTWISE_HPP diff --git a/docker/rocm/migraphx/include/migraphx/op/pooling.hpp b/docker/rocm/migraphx/include/migraphx/op/pooling.hpp deleted file mode 100644 index d696857b2..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/pooling.hpp +++ /dev/null @@ -1,487 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_POOLING_HPP -#define MIGRAPHX_GUARD_OPERATORS_POOLING_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -// The Pooling operator mostly follows the specifications for the Onnx pooling op. -// It assumes an NCHW layout, extended to support any number of spatial dimensions -// from 1 on up; dimensions are -// -struct pooling -{ - // Class members mode, ceil_mode, padding_mode have similar names but refer to separate - // concepts. - pooling_mode mode = {pooling_mode::average}; - - // If the input has rank other than 4 then padding, stride, lengths must all be specified - // since the defaults have 2-dimensions. Exception: padding not required if - // padding_mode != default_ - - // Padding along each spatial input dimension - // Can be ndim or 2*ndim values where ndim is size of lengths - // ndim values means pad the same before and after each dimension - // 2*ndim values contains n pre and then n post padding values - std::vector padding = {0, 0}; - - // Size of stride to take from one placement of the pooling kernel to the next. - // This is distinct from the strides used by the shape class. Must be the same - // ndim as lengths. - std::vector stride = {1, 1}; - - // Spatial dimensions of the pooling kernel or window, - // 2 smaller than the input tensor rank (NCHW layout) - std::vector lengths = {1, 1}; - - // Spacing between the elements of the pooling kernel. Must be the same ndim as lengths. - std::vector dilations = {1, 1}; - - // ceiling mode is a flag affecting output size - // or equivalently, placements of the pooling kernel. - // When true, round the size upwards. When false, round down so that all - // kernel placements fit but some input values may be dropped. - bool ceil_mode = false; - int lp_order = 2; - - // Mode for auto padding. default_ indicates no auto padding. - padding_mode_t padding_mode = padding_mode_t::default_; - - // Global pooling with dynamic shape input - bool dyn_global = false; - - // Whether padding elements are included in the average count - bool count_include_pad = false; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.mode, "mode"), - f(self.padding, "padding"), - f(self.padding_mode, "padding_mode"), - f(self.stride, "stride"), - f(self.lengths, "lengths"), - f(self.dilations, "dilations"), - f(self.ceil_mode, "ceil_mode"), - f(self.count_include_pad, "count_include_pad"), - f(self.lp_order, "lp_order"), - f(self.dyn_global, "dyn_global")); - } - - std::string name() const { return "pooling"; } - - void check_attribute_size() const - { - if(dyn_global) - return; - if((padding_mode != default_ and padding.size() != stride.size() and - (padding.size()) != stride.size() * 2) or - stride.size() != lengths.size() or dilations.size() != lengths.size()) - { - MIGRAPHX_THROW("POOLING: inconsistent attribute sizes"); - } - - const auto is_zero = [](auto el) { return el == 0; }; - if(std::any_of(lengths.begin(), lengths.end(), is_zero) or - std::any_of(stride.begin(), stride.end(), is_zero) or - std::any_of(dilations.begin(), dilations.end(), is_zero)) - { - MIGRAPHX_THROW("POOLING: size 0 pooling kernel or stride or dilations"); - } - } - - size_t kdims() const - { - check_attribute_size(); - return stride.size(); - } - - value attributes() const { return {{"normalize_padding", "padding"}}; } - - inline std::size_t dilate_dim(std::size_t dim, std::size_t dilation) const - { - return 1 + dilation * (dim - 1); - } - - std::vector calc_spatial_dim_out(const std::vector& input_lens, - std::size_t kdims) const - { - std::vector output_lens{}; - for(size_t i = 0; i < kdims; ++i) - { - std::size_t padding_factor = 2 * padding[i]; - if(padding.size() == 2 * kdims) - padding_factor = padding[i] + padding[i + kdims]; - std::size_t dilated_length = dilate_dim(lengths[i], dilations[i]); - std::size_t dim_size; - if(input_lens[i + 2] + padding_factor < dilated_length) - { - if(padding_mode == default_) - MIGRAPHX_THROW("POOLING: not enough padding for the given kernel size"); - // lengths can be legitimately larger only if we're doing auto padding - // with a dynamic shape, in which case given padding is ignored. Set a dummy value. - dim_size = 2; - } - else - { - dim_size = input_lens[i + 2] + padding_factor - dilated_length; - } - std::size_t len = - (ceil_mode) - ? dim_size / stride[i] + - static_cast((dim_size % stride[i] != 0)) // ceil uint divide - : dim_size / stride[i]; // floor divide - output_lens.push_back(len + 1); - } - return output_lens; - } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1).min_ndims(3); - check_attribute_size(); - - const shape& input = inputs.at(0); - auto stride_size = stride.size(); - size_t kdims = input.ndim() - 2; - if(input.ndim() != stride_size + 2) - { - MIGRAPHX_THROW("POOLING: input and attribute size mismatch!"); - } - - if(input.dynamic()) - { - auto input_dyn_dims = input.dyn_dims(); - std::vector output_dyn_dims(input_dyn_dims.begin(), - input_dyn_dims.begin() + 2); - if(dyn_global) - { - for(size_t i = 0; i < kdims; ++i) - { - output_dyn_dims.push_back(shape::dynamic_dimension{1, 1}); - } - return {input.type(), output_dyn_dims}; - } - else if(padding_mode != default_) - { - const size_t num_spatial_dims = inputs[0].ndim() - 2; - const shape& x_shape = inputs[0]; - // same as convolution::dynamic_compute_shape() - - for(std::size_t i = 0; i < num_spatial_dims; ++i) - { - auto ceil_div = [](std::size_t x, std::size_t y) { return (x + y - 1) / y; }; - auto s = stride[i]; - - auto x = x_shape.dyn_dims()[i + 2]; - std::set optimals{}; - std::transform(x.optimals.begin(), - x.optimals.end(), - std::inserter(optimals, optimals.begin()), - [&](auto o) { return ceil_div(o, s); }); - output_dyn_dims.push_back( - shape::dynamic_dimension{ceil_div(x.min, s), ceil_div(x.max, s), optimals}); - } - return {input.type(), output_dyn_dims}; - } - else - { - // does not compute optimals - auto min_spatial_dims = calc_spatial_dim_out(input.min_lens(), kdims); - auto max_spatial_dims = calc_spatial_dim_out(input.max_lens(), kdims); - for(size_t i = 0; i < kdims; ++i) - { - output_dyn_dims.push_back( - shape::dynamic_dimension{min_spatial_dims[i], max_spatial_dims[i], {}}); - } - return {input.type(), output_dyn_dims}; - } - } - else - { - auto input_lens = input.lens(); - - std::vector output_lens(input_lens.begin(), input_lens.begin() + 2); - // Used for when normalize_compute_shape() is called again at model eval time - // for an originally dynamic shape. Kernel shape is not used with dyn_global. - if(dyn_global) - { - for(size_t i = 0; i < kdims; ++i) - { - output_lens.push_back(1); - } - return {input.type(), output_lens}; - } - else - { - auto output_spatial_lens = calc_spatial_dim_out(input_lens, kdims); - output_lens.insert( - output_lens.end(), output_spatial_lens.begin(), output_spatial_lens.end()); - return inputs[0].with_lens(output_lens); - } - } - } - - struct lpnorm_pool - { - int p = 0; - - lpnorm_pool() = delete; - - explicit lpnorm_pool(int x) : p{x} {}; - - template - double init() const - { - return 0.0; - } - - double operator()(double x, double y) const { return x + std::pow(std::abs(y), p); } - - double final(double x, std::size_t) const { return (p == 0) ? 1 : std::pow(x, 1. / p); } - }; - - struct avg_pool - { - template - double init() const - { - return 0.0; - } - - double operator()(double x, double y) const { return x + y; } - - double final(double x, std::size_t y) const { return (y == 0) ? 0.0 : (x / y); } - }; - - struct max_pool - { - template - T init() const - { - return std::numeric_limits::lowest(); - } - - double operator()(double x, double y) const { return std::max(x, y); } - - double final(double x, std::size_t) const { return (x); } - }; - - template - void calc_pooling(const shape& output_shape, - Out& output, - const In& input, - const std::vector& kernel_dims, - const std::vector& padding_vals, - Op op) const - { - auto in_s = input.get_shape(); - auto in_lens = in_s.lens(); - - // For each element of output; i.e., for each placement of pooling kernel... - par_for(output_shape.elements(), [&](auto i) { - auto idx_o = output_shape.multi(i); - auto n_dim = idx_o.size(); - // starting offset of the pooling window - std::vector win_start; - std::vector win_size; - - // For each spatial dimension, find starting and ending index of pooling kernel - for(std::size_t dim = 2; dim < n_dim; ++dim) - { - auto d_2 = dim - 2; - int start = static_cast(idx_o[dim] * stride[d_2]) - - static_cast(padding_vals[d_2]); - int end; - std::size_t dilated_kernel_dim = dilate_dim(kernel_dims[d_2], dilations[d_2]); - // NOLINT - if(count_include_pad and mode == pooling_mode::average) - { - // Even when using padding, if in ceil_mode a window - // could extend beyond the end of both input and - // padding. Clip out-of-bounds indexes but not padding. - - // Check if this kernel extends beyond the padding at end of dimension - end = std::min(start + dilated_kernel_dim, - in_lens[dim] + static_cast(padding_vals[d_2])); - } - else - { - // count_include_pad is false, or for max pooling, clip off padding. - end = std::min(start + dilated_kernel_dim, in_lens[dim]); - } - win_start.push_back(start); - if(end < start) - { - // This error can be caused by misc. bad input combinations - MIGRAPHX_THROW("POOLING: invalid attributes"); - } - win_size.push_back(end - start); - } - - shape win_shape{output_shape.type(), win_size}; - - auto pool_size = win_shape.elements(); - double output_val = op.template init(); - - // for each element in the window... - shape_for_each(win_shape, [&](const auto& idx_w) { - // Skip elements that belong to the dilated area - for(size_t axis = 0; axis < idx_w.size(); ++axis) - { - if(idx_w[axis] % dilations[axis]) - { - pool_size -= 1; - return; - } - } - - // the coordinates of this element - auto idx = idx_o; - - // Add the kernel location idx_w and the offset win_start, for each dimension. - // Negative results are cast to very large unsigned integers. - std::transform(idx_w.begin(), - idx_w.end(), - win_start.begin(), - idx.begin() + 2, - [](auto ii, auto jj) { return ii + jj; }); - // Check if any of coordinates are out of input tensor's range - if(std::equal(idx.begin() + 2, - idx.end(), - in_lens.begin() + 2, - in_lens.end(), - std::less<>{})) - { - output_val = op(output_val, input[idx]); - } - else - { - // this is a padding element. Padding locations - // don't contribute to average or max pooling total but can play in - // lpnorm pooling. - if(mode == pooling_mode::lpnorm) - { - output_val = op(output_val, op.template init()); - } - if(mode == pooling_mode::average and not count_include_pad) - { - // Ignore padding - pool_size -= 1; - } - } - }); - output[i] = Type(op.final(output_val, pool_size)); - }); - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result; - auto input_lens = args[0].get_shape().lens(); - std::vector kernel_dims; - shape output_shape; - // If we have to auto-calculate padding, it will be passed to calc_pooling() as an argument - // instead of the member variable padding. - std::vector temp_padding(padding); - if(dyn_global) - { - // for dynamic GlobalPooling, there's no padding - kernel_dims.insert(kernel_dims.end(), input_lens.begin() + 2, input_lens.end()); - output_shape = dyn_out.computed_shape; - result = argument{dyn_out.computed_shape}; - } - else if((padding_mode != op::padding_mode_t::default_)) - { - // if padding_mode is set, input was a dynamic size. Calculate padded size now. - - // kernel_lens is the same as kernel_dims, but prepended with the 2 non- - // spatial dimensions. For size computations, it's used like the weights - // tensor for convolutions. - std::vector kernel_lens; - kernel_lens.insert(kernel_lens.end(), input_lens.begin(), input_lens.begin() + 2); - kernel_lens.insert(kernel_lens.end(), lengths.begin(), lengths.end()); - kernel_dims = this->lengths; - - auto type = args[0].get_shape().type(); - // dilation not currently supported for pooling, so default to all 1's - temp_padding = calc_dyn_auto_pad( - input_lens, kernel_lens, stride, {1, 1}, bool(padding_mode == op::same_upper)); - - output_shape = compute_padded_pool_shape( - args[0].get_shape(), shape(type, kernel_dims), temp_padding, stride, {1, 1}); - - result = argument(output_shape); - } - else // fixed/static input - { - kernel_dims = this->lengths; - output_shape = dyn_out.computed_shape; - result = argument{dyn_out.computed_shape}; - } - - // Perform the computation and populate result - visit_all(result, args[0])([&](auto output, auto input) { - using type = typename decltype(output)::value_type; - switch(mode) - { - case migraphx::op::pooling_mode::average: - calc_pooling( - output_shape, output, input, kernel_dims, temp_padding, avg_pool{}); - break; - case migraphx::op::pooling_mode::max: - calc_pooling( - output_shape, output, input, kernel_dims, temp_padding, max_pool{}); - break; - case migraphx::op::pooling_mode::lpnorm: - calc_pooling( - output_shape, output, input, kernel_dims, temp_padding, lpnorm_pool{lp_order}); - break; - } - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/pow.hpp b/docker/rocm/migraphx/include/migraphx/op/pow.hpp deleted file mode 100644 index c5da8fb3e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/pow.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_POW_HPP -#define MIGRAPHX_GUARD_OPERATORS_POW_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct pow : binary -{ - auto apply() const - { - return [](auto x, auto y) { return std::pow(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/prefix_scan_op.hpp b/docker/rocm/migraphx/include/migraphx/op/prefix_scan_op.hpp deleted file mode 100644 index c82eabd54..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/prefix_scan_op.hpp +++ /dev/null @@ -1,157 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/** - * Parent struct for prefix scan ops. A prefix scan is a mathematical entity useful - * in parallelizing various computations. Given a list of numbers, a prefix scan - * op returns an equal size list of running totals of the values. Other operations - * besides addition can be supported by child ops. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCAN_OP_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCAN_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Parent struct for prefix scan operations. A prefix scan is equivalent to the C++ - * std::exclusive_scan or std::inclusive_scan. Given a list of numbers, a prefix scan - * sum op returns an equal size list of running totals of the values. Other operations - * besides addition can be supported by their own child ops. - */ -template -struct prefix_scan_op : op_name -{ - int64_t axis; - bool exclusive = false; - bool reverse = false; - - template - static auto reflect(Self& self, F f) - { - return pack( - f(self.axis, "axis"), f(self.exclusive, "exclusive"), f(self.reverse, "reverse")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto s = inputs.front(); - if(s.dynamic()) - { - return s; - } - else if(s.broadcasted()) - { - return {s.type(), s.lens()}; - } - else - { - return s.with_lens(s.lens()); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - shape output_shape(dyn_out.computed_shape); - argument result{output_shape}; - auto s = args[0].get_shape(); - if(s == output_shape) - { - result = args[0].copy(); - } - else - { - visit_all(result, args[0])([&](auto output, auto input) { - par_for(output_shape.elements(), - [&](auto i) { output[output_shape.index(i)] = input[s.index(i)]; }); - }); - s = output_shape; - } - auto slice = shape{s.type(), {s.lens()[axis]}, {s.strides()[axis]}}; - auto lens = s.lens(); - lens[axis] = 1; - auto batch = shape{s.type(), lens, s.strides()}; - auto& self = static_cast(*this); - result.visit([&](auto output) { - using type = decltype(output); - par_for(batch.elements(), [&](auto i) { - auto* start = output.data() + batch.index(i); - type x{slice, start}; - if(reverse) - { - if(exclusive) - { - std::copy(++x.begin(), x.end(), x.begin()); - x.back() = 0; - } - std::partial_sum(std::make_reverse_iterator(x.end()), - std::make_reverse_iterator(x.begin()), - std::make_reverse_iterator(x.end()), - self.op()); - } - else - { - if(exclusive) - { - std::copy_backward(x.begin(), --x.end(), x.end()); - x.front() = 0; - } - std::partial_sum(x.begin(), x.end(), x.begin(), self.op()); - } - }); - }); - - return result; - } - - auto init() const {} - prefix_scan_op() : axis(0) {} - prefix_scan_op(int64_t ax) : axis(ax) {} - prefix_scan_op(int64_t ax, bool excl) : axis(ax), exclusive(excl) {} - prefix_scan_op(int64_t ax, bool excl, bool rev) : axis(ax), exclusive(excl), reverse(rev) {} -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/prefix_scan_sum.hpp b/docker/rocm/migraphx/include/migraphx/op/prefix_scan_sum.hpp deleted file mode 100644 index 2f5d7e28d..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/prefix_scan_sum.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCAN_INCLUSIVE_SUM_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCAN_INCLUSIVE_SUM_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct prefix_scan_sum : prefix_scan_op -{ - prefix_scan_sum() {} - prefix_scan_sum(int64_t ax) : prefix_scan_op(ax) {} - prefix_scan_sum(int64_t ax, bool excl) : prefix_scan_op(ax, excl) {} - prefix_scan_sum(int64_t ax, bool excl, bool rev) : prefix_scan_op(ax, excl, rev) {} - - auto op() const - { - return [](auto x, auto y) { return x + y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/prelu.hpp b/docker/rocm/migraphx/include/migraphx/op/prelu.hpp deleted file mode 100644 index 0f83bd7bf..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/prelu.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_PRELU_HPP -#define MIGRAPHX_GUARD_OPERATORS_PRELU_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct prelu : binary -{ - std::string point_op() const { return "(${0} < 0) ? (${0} * ${1}) : ${0}"; } - auto apply() const - { - return [](auto x, auto slope) { return ((x < 0) ? (x * slope) : x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/quant_convolution.hpp b/docker/rocm/migraphx/include/migraphx/op/quant_convolution.hpp deleted file mode 100644 index 22cbd124b..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/quant_convolution.hpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_QUANT_CONVOLUTION_HPP -#define MIGRAPHX_GUARD_OPERATORS_QUANT_CONVOLUTION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct quant_convolution -{ - std::vector padding = {0, 0}; - std::vector stride = {1, 1}; - std::vector dilation = {1, 1}; - - padding_mode_t padding_mode = default_; - int group = 1; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.padding, "padding"), - f(self.stride, "stride"), - f(self.dilation, "dilation"), - f(self.padding_mode, "padding_mode"), - f(self.group, "group")); - } - - value attributes() const - { - return {{"general_data_type", "convolution"}, {"normalize_padding", "padding"}}; - } - - std::string name() const { return "quant_convolution"; } - - void check_attribute_size() const - { - if((padding.size() != stride.size() and (padding.size() / 2) != stride.size()) or - stride.size() != dilation.size()) - { - MIGRAPHX_THROW("QUANT_CONVOLUTION: inconsistent attribute sizes"); - } - } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(2).same_type().same_ndims().min_ndims(3); - check_attribute_size(); - - const shape& input = inputs.at(0); - const shape& weights = inputs.at(1); - auto t = input.type(); - size_t kdims = input.lens().size() - 2; - if(kdims != this->kdims()) - { - MIGRAPHX_THROW("quant_convolution: input k-dims does not match attribute size"); - } - - // all input type must be int8_type or fp8 types - // output should be float_type - std::set supported_types = fp8_types{}.get(); - supported_types.insert(shape::int8_type); - if(not contains(supported_types, t)) - { - MIGRAPHX_THROW("QUANT_CONVOLUTION: only accept input and weights of type int8 or fp8"); - } - - std::vector output_lens{input.lens()[0], weights.lens()[0]}; - auto padding_size = padding.size(); - for(size_t i = 0; i < kdims; i++) - { - auto padding_factor = 2 * padding[i]; - if(padding_size == 2 * kdims) - padding_factor = padding[i] + padding[i + kdims]; - output_lens.push_back(std::size_t(std::max( - 1, - (input.lens()[i + 2] - (1 + dilation[i] * (weights.lens()[i + 2] - 1)) + - padding_factor) / - stride[i] + - 1))); - } - if(t == shape::int8_type) - { - return inputs[0].with_lens(shape::int32_type, output_lens); - } // else fp8 conv - return inputs[0].with_lens(shape::float_type, output_lens); - } - - size_t kdims() const - { - check_attribute_size(); - return stride.size(); - } - - argument compute(shape output_shape, std::vector args) const - { - argument result{output_shape}; - result.visit([&](auto output) { - visit_all(args[0], args[1])([&](auto input, auto weights) { - migraphx::convolution(output, input, weights, padding, stride, dilation, group); - }); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/quant_dot.hpp b/docker/rocm/migraphx/include/migraphx/op/quant_dot.hpp deleted file mode 100644 index e74daca29..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/quant_dot.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_QUANT_DOT_HPP -#define MIGRAPHX_GUARD_OPERATORS_QUANT_DOT_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct quant_dot -{ - value attributes() const { return {{"general_data_type", "dot"}}; } - - std::string name() const { return "quant_dot"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{{inputs.at(0), inputs.at(1)}, *this}.same_type().has(2); - const shape& a = inputs.at(0); - const shape& b = inputs.at(1); - auto t = a.type(); - std::set supported_types = fp8_types{}.get(); - supported_types.insert(shape::int8_type); - supported_types.insert(shape::uint8_type); - if(not contains(supported_types, t)) - { - MIGRAPHX_THROW("QUANT_DOT: only support data type int8_t, uint8_t and fp8 types"); - } - - if(not std::all_of( - inputs.begin(), inputs.end(), [](auto s) { return s.lens().size() >= 2; })) - { - MIGRAPHX_THROW("QUANT_DOT: dot only accept 2 or more dims operands"); - } - - // only handle the case that the batch size of a and b are the same - if(not std::equal( - a.lens().rbegin() + 2, a.lens().rend(), b.lens().rbegin() + 2, b.lens().rend())) - { - MIGRAPHX_THROW("QUANT_DOT: batch size of A and B mismatch: {" + - to_string_range(a.lens()) + "} x {" + to_string_range(b.lens()) + "}"); - } - - std::size_t dim_0 = a.lens().size() - 2; - std::size_t dim_1 = a.lens().size() - 1; - if(a.lens()[dim_1] != b.lens()[dim_0]) - { - MIGRAPHX_THROW("QUANT_DOT: inner dimensions do not match: {" + - to_string_range(a.lens()) + "} x {" + to_string_range(b.lens()) + "}"); - } - - auto out_lens = a.lens(); - out_lens[dim_1] = b.lens()[dim_1]; - if(contains(fp8_types{}.get(), t)) - { - return {shape::float_type, out_lens}; - } // else int8 gemm - return {shape::int32_type, out_lens}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/quantizelinear.hpp b/docker/rocm/migraphx/include/migraphx/op/quantizelinear.hpp deleted file mode 100644 index 7a0de31cf..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/quantizelinear.hpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_QUANTIZE_LINEAR_HPP -#define MIGRAPHX_GUARD_OPERATORS_QUANTIZE_LINEAR_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { -struct quantizelinear -{ - std::string name() const { return "quantizelinear"; } - std::optional out_type; - - value attributes() const - { - // Note: point_op attribute is not used in this op. Instead, in - // gpu compilation pipeline, rewrite_quantization will be invoked - // from generate_pointwise() to rewrite this op. - return {{"pointwise", true}}; - } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.out_type, "out_type")); - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.same_dims().has(2, 3); - if(inputs[0].type() != inputs[1].type()) - { - MIGRAPHX_THROW("QUANTIZELINEAR: Scales and input must be the same type"); - } - if(inputs.size() == 3) - { - return inputs[0].with_lens(inputs[2].type(), inputs[0].lens()); - } - return inputs[0].with_lens(out_type.value_or(shape::uint8_type), inputs[0].lens()); - } - - argument compute(const shape& output_shape, std::vector args) const - { - auto x = args.at(0); - auto y_scale = args.at(1); - std::vector zeros(output_shape.bytes(), 0); - argument y_zero_point{output_shape, zeros.data()}; - if(args.size() == 3) - { - y_zero_point = args.at(2); - } - argument result{output_shape}; - auto rounding_mode = fegetround(); - fesetround(FE_TONEAREST); - visit_all(result, y_zero_point)([&](auto output, auto zero_pts) { - visit_all(x, y_scale)([&](auto input, auto scales) { - using quant_type = typename decltype(output)::value_type; - auto min_value = std::numeric_limits::lowest(); - auto max_value = std::numeric_limits::max(); - par_for(output_shape.elements(), [&](auto i) { - double quantized = static_cast(std::nearbyint(input[i] / scales[i])) + - static_cast(zero_pts[i]); - output[i] = std::max(static_cast(min_value), - std::min(static_cast(max_value), quantized)); - }); - }); - }); - fesetround(rounding_mode); - return result; - } -}; -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/random_seed.hpp b/docker/rocm/migraphx/include/migraphx/op/random_seed.hpp deleted file mode 100644 index ccd018838..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/random_seed.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_OPERATORS_RANDOM_SEED_HPP -#define MIGRAPHX_GUARD_OPERATORS_RANDOM_SEED_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Generates a random seed for the use of random number generators. Generating the seed - * at runtime guarantees there will be a different random sequence on every execution. - * This operation has no inputs or attributes, and outputs an unsigned integer tensor with - * a single value. - */ -struct random_seed -{ - shape::type_t dtype = shape::type_t::uint64_type; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.dtype, "dtype")); - } - - std::string name() const { return "random_seed"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return shape{dtype}; - } - - argument compute(const shape& output_shape, const std::vector&) const - { - argument result(output_shape); - - result.visit([&](auto output) { output.front() = std::random_device{}(); }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/random_uniform.hpp b/docker/rocm/migraphx/include/migraphx/op/random_uniform.hpp deleted file mode 100644 index f873ae9d3..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/random_uniform.hpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/** - * Random Uniform distribution operator. Given a shape, populate it with random - * values. Calls to random_uniform using the same randomization seed as a - * literal input will - * always generate the same pseudo-random sequence. - * - * Inputs: (1) randomization seed (any type is allowed) - * (2) output buffer argument to be populated. - * - * Attributes: none - * - * Output: Returns the buffer from input #2. - * - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RANDOM_UNIFORM_HPP -#define MIGRAPHX_GUARD_OPERATORS_RANDOM_UNIFORM_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * random_uniform populates the passed shape with random numbers, in a uniform - * distribution. Range for floating-point data types is (0, 1); - * for integer types it is [0, ] - */ -struct random_uniform -{ - // The random_uniform operation needs the random number generator seed - // to be passed as a runtime input. - - std::string name() const { return "random_uniform"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(2); - - return inputs.at(1); - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - // Output goes into the passed buffer, not the shape output. - argument result{dyn_out.computed_shape}; - uint64_t local_seed = args[0].at(0); - std::mt19937 gen(local_seed); - - result.visit([&](auto output) { - using type = typename decltype(output)::value_type; - if constexpr(std::is_integral{}) - { -#ifdef _MSC_VER - // According to the C++ specification, the effect is undefined if the result type - // for the generator is not one of short, int, long, long long, unsigned short, - // unsigned int, unsigned long, or unsigned long long. See - // https://en.cppreference.com/w/cpp/numeric/random/uniform_int_distribution. - if constexpr(sizeof(type) == 1) - { - std::uniform_int_distribution dis{std::numeric_limits::min(), - std::numeric_limits::max()}; - std::generate(output.begin(), output.end(), [&] { return dis(gen); }); - } - else -#endif - { - // default range for all integer types is - // (0, std::uniform_int_distribution::max()). - // Todo: enable different ranges - std::uniform_int_distribution dis; - std::generate(output.begin(), output.end(), [&] { return dis(gen); }); - } - } - else - { - // default real distribution type is double with range (0, 1); - std::uniform_real_distribution<> dis; - std::generate(output.begin(), output.end(), [&] { return dis(gen); }); - } - }); - return result; - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 1; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/recip.hpp b/docker/rocm/migraphx/include/migraphx/op/recip.hpp deleted file mode 100644 index 0947a2ee5..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/recip.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RECIP_HPP -#define MIGRAPHX_GUARD_OPERATORS_RECIP_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct recip : unary -{ - std::string point_op() const { return "1 / ${0}"; } - auto apply() const - { - return [](auto x) { return 1 / x; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_all.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_all.hpp deleted file mode 100644 index 31a50f952..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_all.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_ALL_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_ALL_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_all : reduce_op -{ - reduce_all() {} - reduce_all(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [=](auto x, auto y) { - if(static_cast(x) and static_cast(y)) - return decltype(x){1}; - return decltype(x){0}; - }; - } - - auto init() const { return one(); } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_any.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_any.hpp deleted file mode 100644 index 7d02a6ce8..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_any.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_ANY_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_ANY_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_any : reduce_op -{ - reduce_any() {} - reduce_any(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [=](auto x, auto y) { - if(static_cast(x) or static_cast(y)) - return decltype(x){1}; - return decltype(x){0}; - }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_max.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_max.hpp deleted file mode 100644 index 71f3a9f4e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_max.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_MAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_MAX_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_max : reduce_op -{ - reduce_max() {} - reduce_max(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [=](auto x, auto y) { return x > y ? x : y; }; - } - - auto init() const { return lowest(); } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_mean.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_mean.hpp deleted file mode 100644 index c367f6a97..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_mean.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_MEAN_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_MEAN_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_mean : reduce_op -{ - reduce_mean() {} - reduce_mean(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [](auto x, auto y) { return x + y; }; - } - - auto output(const shape& s) const - { - return [&](auto val) { return val / static_cast(s.elements()); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_min.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_min.hpp deleted file mode 100644 index 0f0510edb..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_min.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_MIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_MIN_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_min : reduce_op -{ - reduce_min() {} - reduce_min(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [=](auto x, auto y) { return x < y ? x : y; }; - } - - auto init() const { return highest(); } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_op.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_op.hpp deleted file mode 100644 index c1fc5e567..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_op.hpp +++ /dev/null @@ -1,273 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_OP_HPP -#define MIGRAPHX_GUARD_OPERATORS_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct lowest -{ - template - operator T() const - { - return std::numeric_limits::lowest(); - } -}; - -struct highest -{ - template - operator T() const - { - return std::numeric_limits::max(); - } -}; - -struct zero -{ - template - operator T() const - { - return T{0}; - } -}; - -struct one -{ - template - operator T() const - { - return T{1}; - } -}; - -template -struct reduce_op : op_name -{ - std::vector axes{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes")); - } - - value attributes() const - { - value normalize; - normalize["axes"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}, {"reduce", true}}; - } - - shape collapse_reduced_axes(const shape& original_shape, - const std::vector& reduce_axes) const - { - auto lens = original_shape.lens(); - for(const auto a : reduce_axes) - { - lens[a] = 1; - } - - return original_shape.with_lens(lens); - } - - // Compute the output shape for cases when the input tensor has a dynamic shape. - // - // If the axes are passed as a variable input(indicated by an empty axes attribute), we cannot - // determine which axes must be collapsed until we see the actual input values, so we must treat - // each axis as potentially collapsable and set its minimum dimension to 1. - shape compute_dynamic_shape(const std::vector& inputs) const - { - const auto& data_shape = inputs[0]; - auto dims = data_shape.dyn_dims(); - if(axes.empty()) - { - for(auto& dim : dims) - { - dim = {1, dim.max}; - } - } - else - { - for(auto a : axes) - { - dims[a] = {1, 1}; - } - } - - return {data_shape.type(), dims}; - } - - // Compute the output shape for cases when the input tensor has a static shape. - // Depending on how axes is passed to the operator the output shape can be either dynamic or - // static. - // - // If the axes are passed as a variable input(indicated by an empty axes attribute), we cannot - // determine which axes must be collapsed until we see the actual input values, so we must treat - // each axis as potentially collapsable, producing a dynamic output shape. - shape compute_static_shape(const std::vector& inputs) const - { - const auto& data_shape = inputs[0]; - if(axes.empty()) - { - std::vector dims(data_shape.ndim()); - auto lens = data_shape.lens(); - std::transform(lens.begin(), lens.end(), dims.begin(), [](auto len) { - return shape::dynamic_dimension{1, len}; - }); - - return {data_shape.type(), std::move(dims)}; - } - else - { - return collapse_reduced_axes(data_shape, axes); - } - } - - /** - * @brief returns a shape in which the axis or axes named - * for reduction by this op are set, to size 1. - * - * @param inputs list of input shapes - * @return shape - */ - shape normalize_compute_shape(std::vector inputs) const - { - auto expected_arg_count = axes.empty() ? 2 : 1; - check_shapes{inputs, *this, true}.has(expected_arg_count); - - if(inputs[0].dynamic()) - { - return compute_dynamic_shape(inputs); - } - else - { - return compute_static_shape(inputs); - } - } - - template - void tune_dims(const std::vector& tuned_axes, - const std::vector& in_lens, - std::vector& out_lens) const - { - for(const auto& axis : tuned_axes) - { - out_lens[axis] = in_lens[axis]; - } - } - - template - void reduce(const tensor_view& input, - const shape& batch_shape, - const std::vector& tuned_axes, - const std::vector& out_idx, - tensor_view& output) const - { - using accumulator = accumulator_type; - auto& self = static_cast(*this); - auto data_idx = out_idx; - accumulator val = self.init(); - shape_for_each(batch_shape, [&](const auto& b_idx) { - this->tune_dims(tuned_axes, b_idx, data_idx); - accumulator x = input(data_idx.begin(), data_idx.end()); - val = self.op()(accumulator{self.input()(x)}, val); - }); - - output(out_idx.begin(), out_idx.end()) = - static_cast(*this).output(batch_shape)(val); - } - - argument reduce(const shape& computed_shape, - const std::vector& reduce_axes, - argument& data_arg) const - { - std::vector batch_lens(computed_shape.ndim(), 1); - auto arg_lens = data_arg.get_shape().lens(); - tune_dims(reduce_axes, arg_lens, batch_lens); - shape batch_shape{computed_shape.type(), batch_lens}; - argument result{computed_shape}; - - visit_all(result, data_arg)([&](auto output, auto input) { - par_for(computed_shape.elements(), [&](auto i) { - auto out_idx = computed_shape.multi(i); - this->reduce(input, batch_shape, reduce_axes, out_idx, output); - }); - }); - - return result; - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - auto&& data_arg = args[0]; - // cppcheck-suppress knownConditionTrueFalse - if(not axes.empty()) - return reduce(dyn_out.computed_shape, axes, data_arg); - - if(args[1].get_shape().elements() == 0) - return args[0]; - - std::vector reduce_axes; - args[1].visit([&](auto&& s) { reduce_axes.assign(s.begin(), s.end()); }); - const auto result_shape = collapse_reduced_axes(data_arg.get_shape(), reduce_axes); - - return reduce(result_shape, reduce_axes, data_arg); - } - - auto init() const { return zero(); } - - auto input() const - { - return [](auto val) { return val; }; - } - - auto output(const shape&) const - { - return [](auto val) { return val; }; - } - - reduce_op() {} - reduce_op(std::vector ax) : axes(std::move(ax)) {} -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_prod.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_prod.hpp deleted file mode 100644 index 97a6b97a1..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_prod.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_PROD_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_PROD_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_prod : reduce_op -{ - reduce_prod() {} - reduce_prod(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [=](auto x, auto y) { return x * y; }; - } - - auto init() const { return one(); } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reduce_sum.hpp b/docker/rocm/migraphx/include/migraphx/op/reduce_sum.hpp deleted file mode 100644 index 8797ff8bd..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reduce_sum.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REDUCE_SUM_HPP -#define MIGRAPHX_GUARD_OPERATORS_REDUCE_SUM_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reduce_sum : reduce_op -{ - reduce_sum() {} - reduce_sum(std::vector ax) : reduce_op(std::move(ax)) {} - - auto op() const - { - return [=](auto x, auto y) { return x + y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/relu.hpp b/docker/rocm/migraphx/include/migraphx/op/relu.hpp deleted file mode 100644 index fddd6bfdc..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/relu.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RELU_HPP -#define MIGRAPHX_GUARD_OPERATORS_RELU_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct relu : unary -{ - std::string point_op() const { return "${function:max}(decltype(${0}){0}, ${0})"; } - auto apply() const - { - return [](auto x) { return std::max(decltype(x){0}, x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reshape.hpp b/docker/rocm/migraphx/include/migraphx/op/reshape.hpp deleted file mode 100644 index 18824a7d8..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reshape.hpp +++ /dev/null @@ -1,232 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RESHAPE_HPP -#define MIGRAPHX_GUARD_OPERATORS_RESHAPE_HPP - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * 1 input version: - * reshape(input_data) - * this.dims = output_dims - * Makes a copy of input_data to the output shape. - * - * 2 input version: - * reshape(input_data, output_buffer) - * this.dims = unset - * Copies input_data to output_buffer; output_buffer already has the output shape. - * This version will not fail gracefully if the input shape and output_buffer shape are - * incompatible. There's a throw that will catch when the number of elements do not match at - * runtime. This version should only be used for dynamic reshapes (output dimensions only known at - * runtime). If output_buffer has a static shape during compile/parse, you can use the 1 input - * version. - */ -struct reshape -{ - std::vector dims; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.dims, "dims")); - } - - std::string name() const { return "reshape"; } - - // Assumes that the shape from the `dims` attribute will be valid at run-time. - // Makes no checks for the validity of the `dims` attribute for the given input shape. - shape dyn_1arg_compute_shape(shape s0) const - { - auto input_dyn_dims = s0.dyn_dims(); - const auto neg_dim_num = - std::distance(this->dims.begin(), std::find(this->dims.begin(), this->dims.end(), -1)); - const bool has_negative_dim_attr = neg_dim_num < dims.size(); - // construct output dynamic shape from dims attribute - std::vector output_dyn_dims(dims.size()); - // NOTE: input_dyn_dims.size() may not equal dims.size() - for(std::size_t i = 0; i < dims.size(); ++i) - { - auto d = dims.at(i); - if(d == 0) - { - output_dyn_dims.at(i) = input_dyn_dims.at(i); - } - else if(d == -1) - { - output_dyn_dims.at(i) = {1, 1}; - } - else - { - std::size_t u_dim = d; - output_dyn_dims.at(i) = {u_dim, u_dim}; - } - } - - if(has_negative_dim_attr) - { - // comparing the -1 dimension against the other dimensions - - // accumulate the minimum and maximum elements in the dimensions before the -1 dimension - std::size_t min_cur_elements = 1; - std::size_t max_cur_elements = 1; - for(const auto& dd : output_dyn_dims) - { - min_cur_elements = mul_sat(min_cur_elements, dd.min); - max_cur_elements = mul_sat(max_cur_elements, dd.max); - } - // accumulate the elements in the input dimensions - std::size_t min_input_elements = 1; - std::size_t max_input_elements = 1; - for(const auto& dd : input_dyn_dims) - { - min_input_elements = mul_sat(min_input_elements, dd.min); - max_input_elements = mul_sat(max_input_elements, dd.max); - } - - // maximum dimensions should never accumulate to zero - assert(max_cur_elements != 0); - - std::size_t max_int = std::numeric_limits::max(); - // handle 0 dimension value (keep unknown lower bound) - std::size_t min_dim = - (min_cur_elements == 0) ? 0 : min_input_elements / min_cur_elements; - // handle maximum dimension value (keep unknown upper bound) - std::size_t max_dim = - (max_cur_elements == max_int) ? max_int : max_input_elements / max_cur_elements; - shape::dynamic_dimension x_dd = {min_dim, max_dim}; - output_dyn_dims.at(neg_dim_num) = x_dd; - } - return {s0.type(), output_dyn_dims}; - } - - shape static_compute_shape(std::vector inputs, std::size_t n_neg_dims) const - { - check_shapes{inputs, *this}.has(1); - auto&& idims = inputs.front().lens(); - std::vector rdims(dims.begin(), dims.end()); - - for(std::size_t i = 0; i < dims.size(); i++) - { - if(dims[i] == 0) - rdims[i] = idims[i]; - - // convert -1 to 1 for rdims since rdims uses size_t (-1 is max_int for size_t) - if(dims[i] == -1) - rdims[i] = 1; - } - - if(n_neg_dims > 0) - { - size_t missing_dim = - inputs.front().elements() / - std::accumulate(rdims.begin(), rdims.end(), 1, std::multiplies()); - for(std::size_t i = 0; i < rdims.size(); i++) - { - if(dims[i] == -1) - rdims[i] = missing_dim; - } - } - - auto s = shape{inputs.front().type(), rdims}; - - if(s.elements() != inputs.front().elements()) - MIGRAPHX_THROW("Reshape: Wrong number of elements for reshape: reshape has " + - std::to_string(s.elements()) + " elements whereas the input has " + - std::to_string(inputs.front().elements())); - - return s; - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1, 2); - - auto n_neg_dims = std::count(dims.begin(), dims.end(), -1); - if(n_neg_dims > 1) - MIGRAPHX_THROW("Reshape: Dimensions for reshape can only have one -1 dim"); - - auto s0 = inputs.front(); - if(inputs.size() == 1) - { - if(s0.dynamic()) - { - return dyn_1arg_compute_shape(s0); - } - else - { - return static_compute_shape(inputs, n_neg_dims); - } - } - else - { - return inputs.back(); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - assert(dyn_out.computed_shape.standard()); - if(args.size() == 1) - { - argument result{dyn_out.computed_shape}; - - visit_all(result, args[0])([&](auto output, auto input) { - std::copy(input.begin(), input.end(), output.begin()); - }); - return result; - } - else - { - // 2 arg - if(args[0].get_shape().elements() != args[1].get_shape().elements()) - { - MIGRAPHX_THROW("Reshape: Number of elements must match at runtime. Input: " + - std::to_string(args[0].get_shape().elements()) + - " Output buffer: " + std::to_string(args[1].get_shape().elements())); - } - visit_all(args[1], args[0])([&](auto output, auto input) { - std::copy(input.begin(), input.end(), output.begin()); - }); - return args[1]; - } - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reshape_lazy.hpp b/docker/rocm/migraphx/include/migraphx/op/reshape_lazy.hpp deleted file mode 100644 index e4dd35849..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reshape_lazy.hpp +++ /dev/null @@ -1,326 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RESHAPE_LAZY_HPP -#define MIGRAPHX_GUARD_OPERATORS_RESHAPE_LAZY_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reshape_lazy -{ - std::vector dims; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.dims, "dims")); - } - - value attributes() const { return {{"require_std_shape", true}}; } - - std::string name() const { return "reshape_lazy"; } - - shape dyn_compute_shape(shape s0) const - { - auto dyn_dims = s0.dyn_dims(); - auto num_not_fixed = std::count_if( - dyn_dims.cbegin(), dyn_dims.cend(), [](auto dd) { return not dd.is_fixed(); }); - if(num_not_fixed != 1) - { - MIGRAPHX_THROW("reshape_lazy: Only supports one non-fixed dynamic_dimension"); - } - // track number of fixed elements in input and output - std::size_t num_dims_ele = 1; - std::size_t num_dd_ele = 1; - for(std::size_t i = 0; i < dyn_dims.size(); ++i) - { - if(dyn_dims[i].is_fixed()) - { - num_dims_ele *= dims[i]; - num_dd_ele *= dyn_dims[i].min; - } - else - { - if(dims[i] != 0 and dims[i] != -1) - { - MIGRAPHX_THROW( - "reshape_lazy: Non-fixed dynamic_dimension doesn't match with 0 or -1 " - "output dimension"); - } - } - } - if(num_dims_ele != num_dd_ele) - { - MIGRAPHX_THROW("reshape_lazy: Number of fixed elements must match. Input: " + - std::to_string(num_dd_ele) + " Output: " + std::to_string(num_dims_ele)); - } - // construct output dynamic shape from dims attribute - std::vector output_dyn_dims(dims.size()); - std::transform(dims.cbegin(), - dims.cend(), - dyn_dims.cbegin(), - output_dyn_dims.begin(), - [](std::size_t dim, auto dyn_dim) { - if(not dyn_dim.is_fixed()) - return dyn_dim; - return shape::dynamic_dimension{dim, dim}; - }); - return {s0.type(), output_dyn_dims}; - } - - template - static auto compute_end_dim(Iterator start, Iterator last, std::size_t dim) - { - std::size_t x = 1; - auto it = std::find_if(start, last, [&](auto i) { - x *= i; - return x >= dim; - }); - if(x != dim) - return start; - return it; - } - - template - static OptionalPair try_merge_pairs(OptionalPair p2, OptionalPair p1) - { - if(not p1.has_value()) - return nullopt; - if(not p2.has_value()) - return nullopt; - auto dim1 = p1->first; - auto dim2 = p2->first; - auto stride1 = p1->second; - auto stride2 = p2->second; - auto elements = dim1 * dim2; - // Transposed - if(stride2 > stride1) - return nullopt; - // Broadcasted check to avoid division by zero - if(stride2 == 0) - { - if(stride1 == 0) - return {{elements, 0}}; - return nullopt; - } - if(stride1 % stride2 != 0) - return nullopt; - auto space = (stride1 * dim1 + stride2 * dim2 - stride1) / stride2; - // Nonpacked - if(space != elements) - return nullopt; - return {{elements, stride2}}; - } - - template - static optional merge_strides(DimIterator dim_start, - DimIterator dim_last, - StrideIterator stride_start, - StrideIterator stride_last) - { - if(dim_start == dim_last) - return nullopt; - (void)stride_start; // Is only used in the assert - assert(std::distance(dim_start, dim_last) == std::distance(stride_start, stride_last)); - auto make_pair_optional = [&](auto dim, auto stride) { - return std::make_optional(std::make_pair(dim, stride)); - }; - auto dim_stride_pair = - std::inner_product(std::make_reverse_iterator(dim_last - 1), - std::make_reverse_iterator(dim_start), - std::make_reverse_iterator(stride_last - 1), - make_pair_optional(*std::prev(dim_last), *std::prev(stride_last)), - MIGRAPHX_LIFT(try_merge_pairs), - make_pair_optional); - if(not dim_stride_pair.has_value()) - return nullopt; - return dim_stride_pair->second; - } - - template - static auto can_strides_merge(DimIterator dim_start, - DimIterator dim_last, - StrideIterator stride_start, - StrideIterator stride_last) - { - return merge_strides(dim_start, dim_last, stride_start, stride_last).has_value(); - } - - // This will attempt to alias the dimensions of the input shape to the lens of - // `rdims`. If this can't be done without changing memory layout then it - // will return nullopt - static optional reshape_lazy_dims(const shape& input, - const std::vector& rdims) - { - if(input.standard()) - return shape{input.type(), rdims}; - - const auto& idims = input.lens(); - const auto& istrides = input.strides(); - - std::vector rstrides; - std::size_t i = 0; - std::size_t r = 0; - while(i < idims.size() and r < rdims.size()) - { - auto idim = idims[i]; - auto rdim = rdims[r]; - if(rdim == idim) - { - rstrides.push_back(istrides[i]); - } - // squeeze - else if(rdim > idim) - { - auto start = idims.begin() + i; - auto it = compute_end_dim(start, idims.end(), rdim); - if(it == start) - return nullopt; - auto n = it - start; - assert((i + n) <= istrides.size()); - if(not can_strides_merge( - start, it + 1, istrides.begin() + i, istrides.begin() + i + n + 1)) - return nullopt; - i += n; - rstrides.push_back(istrides[i]); - } - // unsqueeze - else // if(rdim < idim) - { - auto start = rdims.begin() + i; - auto it = compute_end_dim(start, rdims.end(), idim); - if(it == start) - return nullopt; - auto n = it - start; - assert((r + n) <= rdims.size()); - auto stride = istrides[i] * idim; - std::for_each(start, it + 1, [&](auto dim) { - stride /= dim; - rstrides.push_back(stride); - }); - r += n; - } - i++; - r++; - } - - // Handle trailing 1s - if(rstrides.size() < rdims.size() and not rstrides.empty()) - { - auto stride = rstrides.back(); - for(auto d : range(rdims.begin() + rstrides.size(), rdims.end())) - { - if(d != 1) - return nullopt; - rstrides.push_back(stride); - } - } - - if(rdims.size() != rstrides.size()) - return nullopt; - - return shape{input.type(), rdims, rstrides}; - } - - shape static_compute_shape(std::vector inputs, std::size_t n_neg_dims) const - { - check_shapes{inputs, *this}.has(1); - auto&& idims = inputs.front().lens(); - std::vector rdims(dims.begin(), dims.end()); - - for(std::size_t i = 0; i < dims.size(); i++) - { - if(dims[i] == 0) - rdims[i] = idims[i]; - - // since rdims using size_t type, -1 is the max value - // is size_t that cause later compuation incorrect - if(dims[i] == -1) - rdims[i] = 1; - } - - if(n_neg_dims > 0) - { - size_t missing_dim = - inputs.front().elements() / - std::accumulate(rdims.begin(), rdims.end(), 1, std::multiplies()); - for(std::size_t i = 0; i < rdims.size(); i++) - { - if(dims[i] == -1) - rdims[i] = missing_dim; - } - } - - auto s = reshape_lazy_dims(inputs.front(), rdims); - if(not s.has_value()) - MIGRAPHX_THROW("reshape_lazy on axis that is not packed."); - - if(s->elements() != inputs.front().elements()) - MIGRAPHX_THROW( - "reshape_lazy: Wrong number of elements for reshape_lazy: reshape_lazy has " + - std::to_string(s->elements()) + " elements whereas the input has " + - std::to_string(inputs.front().elements())); - - assert(s->bytes() == inputs.front().bytes()); - return *s; - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto n_neg_dims = std::count(dims.begin(), dims.end(), -1); - if(n_neg_dims > 1) - MIGRAPHX_THROW("reshape_lazy: Dimensions for reshape_lazy can only have one -1 dim"); - auto s0 = inputs[0]; - if(s0.dynamic()) - { - return dyn_compute_shape(s0); - } - else - { - return static_compute_shape(inputs, n_neg_dims); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/resize.hpp b/docker/rocm/migraphx/include/migraphx/op/resize.hpp deleted file mode 100644 index 59a4fde69..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/resize.hpp +++ /dev/null @@ -1,313 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_OPERATORS_RESIZE_HPP -#define MIGRAPHX_GUARD_OPERATORS_RESIZE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * The Resize operation mirrors the Onnx Resize operation with some differences. - * Currently, only Nearest mode is supported. "Axes" and "ROI" attributes not recognized. - * - * Accepts either one or two runtime inputs. - * Input 0 - data to be resized - * Input 1 - sizes or scales. If data type is uint64, Input 1 is interpreted as output sizes; - * otherwise as scaling factors. - * - * If the second input is not used, either a "sizes" or "scales" attribute must be - * provided. - */ -struct resize -{ - // Selects one of several integer rounding rules for use with Nearest mode. - // - // Returns a lambda that returns size_t. - static auto& get_nearest_op(const std::string& near_mode) - { - using nearest_op = std::function; - static std::unordered_map const nearest_ops = { - {"round_prefer_floor", - [=](std::size_t d_in, double val) { - val = std::max(0.0, std::min(d_in - 1.0, val)); - return static_cast(std::ceil((val - 0.5))); - }}, - {"round_prefer_ceil", - [=](std::size_t d_in, double val) { - val = std::max(0.0, std::min(d_in - 1.0, val)); - return static_cast(std::round((val))); - }}, - {"floor", - [=](std::size_t d_in, double val) { - val = std::max(0.0, std::min(d_in - 1.0, val)); - return static_cast(std::floor((val))); - }}, - {"ceil", [=](std::size_t d_in, double val) { - val = std::max(0.0, std::min(d_in - 1.0, val)); - return static_cast(std::ceil((val))); - }}}; - - if(not contains(nearest_ops, near_mode)) - { - MIGRAPHX_THROW("RESIZE: nearest_mode " + near_mode + " not supported!"); - } - - return nearest_ops.at(near_mode); - } - - // Selects one of several rules for converting a coordinate by a scaling factor. - // These rules differ in how they account for subpixel distances and end - // values. They apply to all modes. - // - // Returns a lambda that returns double. - static auto& get_original_idx_op(const std::string& s_mode) - { - using original_idx_op = - std::function; - static std::unordered_map const idx_ops = { - {"half_pixel", - [=](std::size_t, std::size_t, std::size_t idx, double scale) { - return (idx + 0.5) / scale - 0.5; - }}, - {"pytorch_half_pixel", - [=](std::size_t, std::size_t l_out, std::size_t idx, double scale) { - return l_out > 1 ? (idx + 0.5) / scale - 0.5 : 0.0; - }}, - {"align_corners", - [=](std::size_t l_in, std::size_t l_out, std::size_t idx, double) { - return (l_out == 1) ? 0.0 : (1.0 * idx * (l_in - 1.0) / (l_out - 1.0)); - }}, - {"asymmetric", - [=](std::size_t, std::size_t, std::size_t idx, double scale) { return idx / scale; }}, - {"tf_half_pixel_for_nn", [=](std::size_t, std::size_t, std::size_t idx, double scale) { - return (idx + 0.5) / scale; - }}}; - - if(not contains(idx_ops, s_mode)) - { - MIGRAPHX_THROW("RESIZE: coordinate_transformation_mode " + s_mode + " not supported!"); - } - - return idx_ops.at(s_mode); - } - - std::vector scales; - std::vector sizes; - // what integer rounding rule to use with Nearest mode. - std::string nearest_mode{"floor"}; - // Resizing modes. 1: nearest 2: bilinear/linear 3: cubic - // Only "nearest" currently supported. - std::string mode{"nearest"}; - // What floating-point conversion rule to use (any resizing mode) - std::string coordinate_transformation_mode; - - std::string name() const { return "resize"; } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.scales, "scales"), - f(self.sizes, "sizes"), - f(self.nearest_mode, "nearest_mode"), - f(self.mode, "mode"), - f(self.coordinate_transformation_mode, "coordinate_transformation_mode")); - } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1, 2); - - if(mode != "nearest") - MIGRAPHX_THROW("RESIZE: Only Nearest mode is supported"); - - // Inputs are X, sizes or scale, ROI and axes not supported. - if(inputs.size() == 1) - { - if(inputs.front().dynamic()) - { - // Not supported at this point--needs different validity checking - MIGRAPHX_THROW("RESIZE: Single dynamic input not supported"); - } - auto input_s = inputs.front(); - // No size/scale input. Size/scale must be an attribute and so output will be static. - if((sizes.empty()) == (scales.empty())) - MIGRAPHX_THROW( - "RESIZE: One and only one of sizes or scales attributes must be given"); - if(not sizes.empty()) - { - if(sizes.size() != input_s.ndim()) - MIGRAPHX_THROW("RESIZE: sizes attribute's size must match rank of input X"); - return shape{input_s.type(), sizes}; - } - else - { - if(scales.size() != input_s.ndim()) - MIGRAPHX_THROW("RESIZE: scales attribute's size must match rank of input X"); - std::vector lens; - std::transform(scales.begin(), - scales.end(), - input_s.lens().begin(), - std::back_inserter(lens), - [](auto scale_i, size_t in_len) { - return static_cast(scale_i * in_len); - }); - return shape{input_s.type(), lens}; - } - } - else - { - // 2 inputs: 2nd sets sizes/scales for output. - if(inputs.back().ndim() != 1) - MIGRAPHX_THROW("RESIZE: size/scale input must have rank 1"); - if(inputs.back().dynamic()) - MIGRAPHX_THROW("RESIZE: size/scale input must have static shape"); - - if(inputs.front().ndim() != inputs.back().lens()[0]) - MIGRAPHX_THROW("RESIZE: size/scale input's size must match rank of input X"); - - // Note: a shape with {0, max_val} ranges can't really be allocated. For the reference - // target, this doesn't matter because the real output shape is determined in the - // compute() method. For any other target, there must be a compiler pass that replaces - // this operation with a fixed-size output at runtime. - std::size_t max_val = std::numeric_limits::max(); - std::vector dyn_dims(inputs.back().lens().at(0), - shape::dynamic_dimension{0, max_val}); - return {inputs.front().type(), dyn_dims}; - } - } - - argument compute(const migraphx::shape&, std::vector args) const - { - auto in_lens = args[0].get_shape().lens(); - std::vector out_lens(in_lens.size()); - - // Scales are either given, or calculated from output shape - std::vector vec_scale(in_lens.size(), 1.0f); - - if(args.size() == 1) - { - // single input argument; sizes or scales is constant. - // In practice, the input is never a dynamic shape. - if(not sizes.empty()) - { - out_lens = sizes; - // compute scales - std::transform(out_lens.begin(), - out_lens.end(), - in_lens.begin(), - vec_scale.begin(), - [](size_t out_len, size_t in_len) { - return (in_len == 0 ? 1.f - : static_cast(out_len) / in_len); - }); - } - else - { - vec_scale = this->scales; - // compute output sizes - std::transform(in_lens.begin(), - in_lens.end(), - scales.begin(), - out_lens.begin(), - [](size_t in_len, auto scale_i) { - return static_cast(scale_i * in_len); - }); - } - } - else - { - // 2 inputs; 2nd input is either sizes or scales. - // First input may be dynamic. - args[1].visit([&](auto input) { - using type = typename decltype(input)::value_type; - if constexpr(std::is_integral{}) - { - // Copy the output size from args[1]. - std::copy(input.begin(), input.end(), out_lens.begin()); - // Deduce the scales for each axis - std::transform( - input.begin(), - input.end(), - in_lens.begin(), - vec_scale.begin(), - [](auto sz, size_t in_len) { return static_cast(sz) / in_len; }); - } - else - { - // read the scale from args[1] - // - std::copy(input.begin(), input.end(), vec_scale.begin()); - // compute the output dimensions from the given scales. This computation - // always rounds down, unlike the internal computation in Nearest mode - // which has several options as given in nearest_mode. - std::transform(input.begin(), - input.end(), - in_lens.begin(), - out_lens.begin(), - [](auto scale_i, size_t in_len) { - return static_cast(scale_i * in_len); - }); - } - }); - } - - shape output_shape = {args[0].get_shape().type(), out_lens}; - argument result{output_shape}; - auto nearest_op = get_nearest_op(nearest_mode); - auto idx_op = get_original_idx_op(coordinate_transformation_mode); - - // Populate each element in output by selecting "nearest" item in input. - visit_all(result, args[0])([&](auto output, auto data) { - migraphx::shape out_comp_shape{data.get_shape().type(), out_lens}; - shape_for_each(out_comp_shape, [&](const auto& out_idx_v, size_t out_idx) { - std::vector in_idx(out_idx_v.size()); - for(auto ii = 0; ii < out_idx_v.size(); ++ii) - { - auto idx_val = idx_op(in_lens[ii], out_lens[ii], out_idx_v[ii], vec_scale[ii]); - in_idx[ii] = nearest_op(in_lens[ii], idx_val); - } - output[out_idx] = data(in_idx.begin(), in_idx.end()); - }); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/reverse.hpp b/docker/rocm/migraphx/include/migraphx/op/reverse.hpp deleted file mode 100644 index 88959da64..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/reverse.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_REVERSE_HPP -#define MIGRAPHX_GUARD_OPERATORS_REVERSE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct reverse -{ - - std::vector axes; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes")); - } - - std::string name() const { return "reverse"; } - - value attributes() const - { - value normalize; - normalize["axes"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1); - return inputs[0].with_lens(inputs[0].lens()); - } - - argument compute(const shape& s, std::vector args) const - { - argument result{s}; - auto lens = s.lens(); - visit_all(result, args.front())([&](auto output, auto input) { - shape_for_each(s, [&](const auto& out_idx_v, size_t out_idx) { - auto in_idx = out_idx_v; - for(const auto& axis : axes) - { - in_idx[axis] = lens[axis] - 1 - out_idx_v[axis]; - } - output[out_idx] = input[s.index(in_idx)]; - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/rnn.hpp b/docker/rocm/migraphx/include/migraphx/op/rnn.hpp deleted file mode 100644 index a94612d78..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/rnn.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RNN_HPP -#define MIGRAPHX_GUARD_OPERATORS_RNN_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct rnn -{ - std::size_t hidden_size = 1; - std::vector actv_funcs{tanh{}, tanh{}}; - rnn_direction direction = rnn_direction::forward; - float clip = 0.0f; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.hidden_size, "hidden_size"), - f(self.actv_funcs, "actv_func"), - f(self.direction, "direction"), - f(self.clip, "clip")); - } - - std::string name() const { return "rnn"; } - shape compute_shape(std::vector inputs) const - { - auto in_dims = inputs[0].lens(); - auto hidden_dims = inputs[2].lens(); - if(hidden_size != hidden_dims[2]) - { - MIGRAPHX_THROW("RNN: hidden size mismatch in attribute and input"); - } - - std::size_t num_directions = 1; - if(direction == rnn_direction::bidirectional) - { - num_directions = 2; - } - - if(num_directions != hidden_dims[0]) - { - MIGRAPHX_THROW("RNN: num_direction mismatch in attribute and input"); - } - - std::vector out_dims(in_dims); - out_dims.insert(out_dims.begin() + 1, num_directions); - out_dims.back() = hidden_size; - - return {inputs[0].type(), out_dims}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/rnn_last_cell_output.hpp b/docker/rocm/migraphx/include/migraphx/op/rnn_last_cell_output.hpp deleted file mode 100644 index 67159fa37..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/rnn_last_cell_output.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RNN_LAST_CELL_OUTPUT_HPP -#define MIGRAPHX_GUARD_OPERATORS_RNN_LAST_CELL_OUTPUT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct rnn_last_cell_output -{ - std::string name() const { return "rnn_last_cell_output"; } - - shape compute_shape(std::vector inputs) const - { - auto dims = inputs[0].lens(); - - // remove the first dimension, remaing are output shape - dims.erase(dims.begin()); - return {inputs[0].type(), dims}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/rnn_last_hs_output.hpp b/docker/rocm/migraphx/include/migraphx/op/rnn_last_hs_output.hpp deleted file mode 100644 index bb075a3ff..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/rnn_last_hs_output.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RNN_LAST_HS_OUTPUT_HPP -#define MIGRAPHX_GUARD_OPERATORS_RNN_LAST_HS_OUTPUT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct rnn_last_hs_output -{ - std::string name() const { return "rnn_last_hs_output"; } - - shape compute_shape(std::vector inputs) const - { - auto dims = inputs[0].lens(); - - // remove the first dimension, remaing are output shape - dims.erase(dims.begin()); - return {inputs[0].type(), dims}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/rnn_var_sl_last_output.hpp b/docker/rocm/migraphx/include/migraphx/op/rnn_var_sl_last_output.hpp deleted file mode 100644 index 5125c7caa..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/rnn_var_sl_last_output.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RNN_VAR_SL_LAST_OUTPUT_HPP -#define MIGRAPHX_GUARD_OPERATORS_RNN_VAR_SL_LAST_OUTPUT_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct rnn_var_sl_last_output -{ - rnn_direction direction = rnn_direction::forward; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.direction, "direction")); - } - - std::string name() const { return "rnn_var_sl_last_output"; } - - shape compute_shape(std::vector inputs) const - { - auto dims = inputs[0].lens(); - - // remove the first dimension, remaing are output shape - dims.erase(dims.begin()); - return {inputs[0].type(), dims}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/rnn_variable_seq_lens.hpp b/docker/rocm/migraphx/include/migraphx/op/rnn_variable_seq_lens.hpp deleted file mode 100644 index 97904b090..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/rnn_variable_seq_lens.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RNN_VARIABLE_SEQ_LENS_HPP -#define MIGRAPHX_GUARD_OPERATORS_RNN_VARIABLE_SEQ_LENS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct rnn_var_sl_shift_output -{ - std::string output_name = "hidden_states"; - rnn_direction direction = rnn_direction::forward; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.output_name, "output_name"), f(self.direction, "direction")); - } - - std::string name() const { return "rnn_var_sl_shift_output"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(2); - return inputs[0]; - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - int64_t max_len = output_shape.lens()[0]; - visit_all(result, args[0])([&](auto output, auto input) { - using value_type = typename decltype(output)::value_type; - args[1].visit([&](auto seq_lens) { - par_for(output_shape.elements(), [&](auto i) { - auto idx = output_shape.multi(i); - auto batch_id = idx[2]; - auto d = idx[1]; - auto t = idx[0]; - auto sl = seq_lens[batch_id]; - value_type val = value_type{0}; - if(t < sl) - { - auto in_idx = idx; - int offset = (direction == rnn_direction::reverse or d == 1) ? 1 : 0; - in_idx[0] += offset * (max_len - sl); - val = input(in_idx.begin(), in_idx.end()); - } - output(idx.begin(), idx.end()) = val; - }); - }); - }); - - return result; - } -}; - -struct rnn_var_sl_shift_sequence -{ - std::string name() const { return "rnn_var_sl_shift_sequence"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(2); - return inputs[0]; - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - int64_t max_len = output_shape.lens()[0]; - visit_all(result, args[0])([&](auto output, auto input) { - using value_type = typename decltype(output)::value_type; - args[1].visit([&](auto seq_lens) { - par_for(output_shape.elements(), [&](auto i) { - auto idx = output_shape.multi(i); - auto b = idx[1]; - auto t = idx[0]; - auto sl = seq_lens[b]; - value_type val = value_type{0}; - if(t >= max_len - sl) - { - auto in_idx = idx; - in_idx[0] -= (max_len - sl); - val = input(in_idx.begin(), in_idx.end()); - } - output(idx.begin(), idx.end()) = val; - }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/roialign.hpp b/docker/rocm/migraphx/include/migraphx/op/roialign.hpp deleted file mode 100644 index d66e8f0fe..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/roialign.hpp +++ /dev/null @@ -1,292 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_ROIALIGN_HPP -#define MIGRAPHX_GUARD_OPERATORS_ROIALIGN_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct roialign -{ - std::string coord_trans_mode = "half_pixel"; - pooling_mode mode = {pooling_mode::average}; - int64_t output_height = 1; - int64_t output_width = 1; - int64_t sampling_ratio = 0; - float spatial_scale = 1.0f; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.coord_trans_mode, "coordinate_transformation_mode"), - f(self.mode, "mode"), - f(self.output_height, "output_height"), - f(self.output_width, "output_width"), - f(self.sampling_ratio, "sampling_ratio"), - f(self.spatial_scale, "spatial_scale")); - } - - std::string name() const { return "roialign"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(3); - auto x_lens = inputs.at(0).lens(); - auto roi_lens = inputs.at(1).lens(); - auto bi_lens = inputs.at(2).lens(); - auto type = inputs.at(0).type(); - - // check input correct - if(bi_lens.size() != 1) - { - MIGRAPHX_THROW("ROIALIGN: batch indices should be 1 dimension!"); - } - - if(roi_lens.size() != 2 or roi_lens.at(1) != 4) - { - MIGRAPHX_THROW( - "ROIALIGN: rois should be 2 dimensions, and the second dim should be 4!"); - } - - if(roi_lens.front() != bi_lens.front()) - { - MIGRAPHX_THROW("ROIALIGN: rois and batch indices inputs should have the same number!"); - } - - std::vector out_lens = x_lens; - out_lens[0] = roi_lens[0]; - out_lens[2] = output_height; - out_lens[3] = output_width; - - return {type, out_lens}; - } - - struct pos_weight - { - // neighbor indices for the bilinear interpolation - std::array pos = {0, 0, 0, 0}; - // neighbor weights for the bilinear interpolation - std::array w = {0.0f, 0.0f, 0.0f, 0.0f}; - }; - - auto calc_pos_weight(const std::array& dims, - const shape& comp_s, - const std::array& roi_start, - const std::array& bin_size, - const std::array& bin_grid_size) const - { - std::vector results(bin_grid_size[0] * bin_grid_size[1] * output_height * - output_width); - shape_for_each(comp_s, [&](const auto& idx_v, size_t index) { - std::array p = {idx_v[0], idx_v[1]}; - std::array i = {idx_v[2], idx_v[3]}; - - std::array xy{}; - std::array low{}; - std::array high{}; - for(auto ii : range(p.size())) - { - xy[ii] = roi_start[ii] + p[ii] * bin_size[ii] + - (i[ii] + .5f) * bin_size[ii] / bin_grid_size[ii]; - xy[ii] = (coord_trans_mode == "half_pixel") ? (xy[ii] - 0.5f) : xy[ii]; - if(xy[ii] < -1.0 or xy[ii] > dims[ii]) - { - results[index] = pos_weight{}; - return; - } - - xy[ii] = std::max(xy[ii], 0.0f); - low[ii] = xy[ii]; - high[ii] = low[ii] + 1; - if(low[ii] >= dims[ii] - 1) - { - xy[ii] = high[ii] = low[ii] = dims[ii] - 1; - } - } - - results[index].pos = {low[0] * dims[1] + low[1], - low[0] * dims[1] + high[1], - high[0] * dims[1] + low[1], - high[0] * dims[1] + high[1]}; - - float ly = xy[0] - low[0]; - float lx = xy[1] - low[1]; - float hy = 1.0f - ly; - float hx = 1.0f - lx; - - // save weights and indeces - results[index].w = {hy * hx, hy * lx, ly * hx, ly * lx}; - }); - - return results; - } - - struct max_pool - { - double init() { return std::numeric_limits::lowest(); } - - double operator()(double x, double y) { return std::max(x, y); } - - double final(double x, std::size_t) { return (x); } - }; - - struct avg_pool - { - double init() { return 0.0; } - - double operator()(double x, double y) { return x + y; } - - double final(double x, std::size_t y) { return (y == 0) ? 0.0 : (x / y); } - }; - - template - std::tuple calc_pooling(const T& data, - const std::array& bin_grid_size, - const std::vector& pos_weights, - int64_t index, - Op op) const - { - double output_val = op.init(); - const int64_t count = bin_grid_size[0] * bin_grid_size[1]; - dfor(bin_grid_size[0], bin_grid_size[1])([&](auto, auto) { - const auto& pc = pos_weights[index]; - std::array wv; - std::transform( - pc.w.begin(), pc.w.end(), pc.pos.begin(), wv.begin(), [&](auto w, auto pos) { - return *(data + pos) * w; - }); - output_val = std::accumulate(wv.begin(), wv.end(), output_val, op); - index += 1; - }); - - output_val = op.final(output_val, count); - - return {output_val, index}; - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - const auto& out_lens = output_shape.lens(); - int64_t n_rois = out_lens[0]; - std::size_t channels = out_lens[1]; - // output dims of height and width, in all 2-dim arrays, the first dim - // is for height and second dim is for width - std::array out_dims = {out_lens[2], out_lens[3]}; - const auto& x_lens = args.at(0).get_shape().lens(); - // input dims of height and width - std::array in_dims = {x_lens[2], x_lens[3]}; - auto roi_s = args.at(1).get_shape(); - - visit_all(result, args.at(0), args.at(1))([&](auto output, auto x, auto roi) { - const auto* batch_indices = args.at(2).cast(); - par_for(n_rois, [&](auto n) { - const auto bottom_data = x.begin(); - const auto roi_batch_ind = batch_indices[n]; - // Do not using rounding; this implementation detail is critical - std::array roi_starts = { - static_cast(roi[roi_s.index({n, 1})] * spatial_scale), - static_cast(roi[roi_s.index({n, 0})] * spatial_scale)}; - std::array roi_ends = { - static_cast(roi[roi_s.index({n, 3})] * spatial_scale), - static_cast(roi[roi_s.index({n, 2})] * spatial_scale)}; - - // Force malformed ROIs to be 1x1 - std::array roi_size{}; - std::array bin_size{}; - std::array bin_grid_size{}; - - for(auto ii : range(roi_size.size())) - { - roi_size[ii] = roi_ends[ii] - roi_starts[ii]; - roi_size[ii] = std::max(roi_size[ii], 1.0f); - - bin_size[ii] = roi_size[ii] / out_dims[ii]; - bin_grid_size[ii] = (sampling_ratio > 0) - ? sampling_ratio - : std::ceil(roi_size[ii] / out_dims[ii]); - } - - // we want to precalculate indices and weights shared by all channels, - // this is the key point of optimization - std::vector comp_lens = { - out_dims[0], out_dims[1], bin_grid_size[0], bin_grid_size[1]}; - shape comp_s{shape::float_type, comp_lens}; - auto pre_calc = - this->calc_pos_weight(in_dims, comp_s, roi_starts, bin_size, bin_grid_size); - - std::vector comp_lens1 = {channels, out_dims[0], out_dims[1]}; - shape comp_s1{migraphx::shape::float_type, comp_lens1}; - std::vector vec_index(channels, 0); - shape_for_each(comp_s1, [&](const auto& idx) { - auto c = idx[0]; - auto ph = idx[1]; - auto pw = idx[2]; - - const auto offset_bottom_data = - bottom_data + static_cast((roi_batch_ind * channels + c) * - in_dims[0] * in_dims[1]); - double output_val; - std::tie(output_val, vec_index[c]) = - (mode == migraphx::op::pooling_mode::average) - ? this->calc_pooling(offset_bottom_data, - bin_grid_size, - pre_calc, - vec_index[c], - avg_pool{}) - : this->calc_pooling(offset_bottom_data, - bin_grid_size, - pre_calc, - vec_index[c], - max_pool{}); - output(n, c, ph, pw) = output_val; - }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/rsqrt.hpp b/docker/rocm/migraphx/include/migraphx/op/rsqrt.hpp deleted file mode 100644 index 7ac1ff7b6..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/rsqrt.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_RSQRT_HPP -#define MIGRAPHX_GUARD_OPERATORS_RSQRT_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct rsqrt : unary -{ - auto apply() const - { - return [](auto x) { return 1 / std::sqrt(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/run_on_target.hpp b/docker/rocm/migraphx/include/migraphx/op/run_on_target.hpp deleted file mode 100644 index 560cf4bae..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/run_on_target.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_RUN_ON_TARGET_HPP -#define MIGRAPHX_GUARD_RTGLIB_RUN_ON_TARGET_HPP -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace op { -struct run_on_target -{ - - std::size_t target_id = 0; - std::string name() const { return "run_on_target"; } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.target_id, "target_id")); - } - - migraphx::shape compute_shape(const std::vector& inputs, - std::vector mods) const - { - if(mods.size() != 1) - { - MIGRAPHX_THROW("RUN_ON_TARGET: must have exactly 1 module argument"); - } - auto* mod_input = mods.front(); - if(inputs.size() != mod_input->get_parameter_shapes().size()) - { - MIGRAPHX_THROW("RUN_ON_TARGET: Mismatched number of input parameters"); - } - auto mod_out_shapes = mod_input->get_output_shapes(); - return shape(mod_out_shapes); - } - - migraphx::argument - compute(const migraphx::shape&, - const std::vector& args, - const std::vector& mods, - const std::function( - migraphx::module_ref&, const std::unordered_map&)>& - run) const - - { - std::unordered_map params; - std::set pnames; - const auto* smod = mods.front(); - assert(mods.size() == 1); - auto names = smod->get_parameter_names(); - pnames.insert(names.begin(), names.end()); - assert(pnames.size() == args.size()); - std::transform(pnames.begin(), - pnames.end(), - args.begin(), - std::inserter(params, params.end()), - [](auto&& name, auto&& arg) { return std::make_pair(name, arg); }); - auto* mod = mods.front(); - auto results = run(mod, params); - return migraphx::argument{results}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scalar.hpp b/docker/rocm/migraphx/include/migraphx/op/scalar.hpp deleted file mode 100644 index 9102e476d..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scalar.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCALAR_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCALAR_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scalar -{ - std::vector scalar_bcast_lens; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.scalar_bcast_lens, "scalar_bcst_dims")); - } - - std::string name() const { return "scalar"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1).only_dims(1).nelements(1); - auto t = inputs.at(0).type(); - std::vector strides(scalar_bcast_lens.size(), 0); - return {t, scalar_bcast_lens, strides}; - } - - argument compute(shape output_shape, std::vector args) const - { - return args[0].reshape(output_shape); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scan_slice.hpp b/docker/rocm/migraphx/include/migraphx/op/scan_slice.hpp deleted file mode 100644 index 6bb5e56c4..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scan_slice.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCAN_SLICE_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCAN_SLICE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scan_slice : op_name -{ - int64_t axis = 0; - int64_t direction = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.direction, "direction")); - } - - value attributes() const - { - value normalize_axes = value::object{}; - normalize_axes["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize_axes}}; - } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(2); - auto input_shape = inputs[0]; - auto new_lens = input_shape.lens(); - new_lens[axis] = 1; - - return shape{input_shape.type(), new_lens, input_shape.strides()}; - } - - argument compute(shape output_shape, std::vector args) const - { - auto input = args[0]; - auto input_sh = input.get_shape(); - - int64_t idx; - args[1].visit([&](auto i) { idx = i.front(); }); - const auto max_idx = input_sh.lens()[axis] - 1; - if(idx > max_idx or idx < 0) - MIGRAPHX_THROW("ScanSlice: index {" + std::to_string(idx) + "} out of range [0, " + - std::to_string(max_idx) + "]"); - idx = (1 - direction) * idx + direction * (max_idx - idx); - - auto offset = idx * input_sh.strides().at(axis) * input_sh.type_size(); - return {output_shape, [=] { return input.data() + offset; }}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatter_add.hpp b/docker/rocm/migraphx/include/migraphx/op/scatter_add.hpp deleted file mode 100644 index af7e9d3ec..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatter_add.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_ADD_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_ADD_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatter_add : public scatter_op -{ - auto reduction() const - { - return [](auto& x, const auto& y) { x += y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatter_max.hpp b/docker/rocm/migraphx/include/migraphx/op/scatter_max.hpp deleted file mode 100644 index b1edf4f3e..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatter_max.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_MAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_MAX_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatter_max : public scatter_op -{ - auto reduction() const - { - return [](auto& x, const auto& y) { x = std::max(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatter_min.hpp b/docker/rocm/migraphx/include/migraphx/op/scatter_min.hpp deleted file mode 100644 index d1177c915..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatter_min.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_MIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_MIN_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatter_min : public scatter_op -{ - auto reduction() const - { - return [](auto& x, const auto& y) { x = std::min(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatter_mul.hpp b/docker/rocm/migraphx/include/migraphx/op/scatter_mul.hpp deleted file mode 100644 index 84260670f..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatter_mul.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_MUL_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_MUL_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatter_mul : public scatter_op -{ - auto reduction() const - { - return [](auto& x, const auto& y) { x *= y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatter_none.hpp b/docker/rocm/migraphx/include/migraphx/op/scatter_none.hpp deleted file mode 100644 index 7e52165ff..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatter_none.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_NONE_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_NONE_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatter_none : public scatter_op -{ - auto reduction() const - { - return [](auto& x, const auto& y) { x = y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatter_op.hpp b/docker/rocm/migraphx/include/migraphx/op/scatter_op.hpp deleted file mode 100644 index 7ac6470b8..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatter_op.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_OP_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTER_ELEMENTS_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -// The scatter operator fetches a subset of data given by an index array and then performs a -// reduction operation (add, multiply, or just set the data) on each element returned. We implement -// it as a separate derived struct for each of the three reduction methods. The related operator -// scatterND is a generalization that works on a set of 3 tensors of different ranks. The -// complementary operations are gather/gatherND. - -template -struct scatter_op : op_name -{ - int64_t axis = 0; - // skip scattering indicies that are out of bounds - bool skip_out_of_bounds = false; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.skip_out_of_bounds, "skip_out_of_bounds")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(3); - // If non-packed, this converts to a packed output while preserving permutation of tensor - return inputs.front().with_lens(inputs.front().lens()); - } - - // iterate through items in shape - template - void scatter_reduce_iterate(TensorView0 indices, TensorView1 output, TensorView1 update) const - { - auto ind_s = indices.get_shape(); - auto output_shape = output.get_shape(); - auto axis_dim_size = output_shape.lens()[axis]; - shape_for_each(ind_s, [&](const auto& idx) { - auto out_idx = idx; - - // tensor_view::() invokes indexing logic of - // std::size_t shape::index(std::size_t i) const - auto index = indices(idx.begin(), idx.end()); - - // this addition doesn't necessarily make index positive if index was out of bounds - index = (index < 0) ? index + axis_dim_size : index; - assert(skip_out_of_bounds or index >= 0); - if(skip_out_of_bounds and index < 0) - { - return; - } - out_idx[axis] = index; - // skip index out of bounds if attribute on, else assert - assert(skip_out_of_bounds or output_shape.multi_within_bounds(out_idx)); - if(skip_out_of_bounds) - { - if(not output_shape.multi_within_bounds(out_idx)) - { - return; - } - } - // look up the appropriate locations in output, using idx and out_idx. - // call reduction() method of derived struct to copy and reduce that element - derived().reduction()(output(out_idx.begin(), out_idx.end()), - update(idx.begin(), idx.end())); - return; - }); - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - - visit_all(result, args[0], args[2])([&](auto output, auto data, auto update) { - std::copy(data.begin(), data.end(), output.begin()); - args[1].visit([&](auto indices) { scatter_reduce_iterate(indices, output, update); }); - }); - - return result; - } - - const Derived& derived() const { return static_cast(*this); } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatternd_add.hpp b/docker/rocm/migraphx/include/migraphx/op/scatternd_add.hpp deleted file mode 100644 index a4cc2e4e6..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatternd_add.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTERND_ADD_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTERND_ADD_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatternd_add : scatternd_op -{ - scatternd_add() {} - - auto reduction() const - { - return [](auto& x, const auto& y) { x += y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatternd_max.hpp b/docker/rocm/migraphx/include/migraphx/op/scatternd_max.hpp deleted file mode 100644 index d8b063418..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatternd_max.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTERND_MAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTERND_MAX_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatternd_max : scatternd_op -{ - scatternd_max() {} - - auto reduction() const - { - return [](auto& x, const auto& y) { x = std::max(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatternd_min.hpp b/docker/rocm/migraphx/include/migraphx/op/scatternd_min.hpp deleted file mode 100644 index ec733bab2..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatternd_min.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTERND_MIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTERND_MIN_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatternd_min : scatternd_op -{ - scatternd_min() {} - - auto reduction() const - { - return [](auto& x, const auto& y) { x = std::min(x, y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatternd_mul.hpp b/docker/rocm/migraphx/include/migraphx/op/scatternd_mul.hpp deleted file mode 100644 index d4ee82abe..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatternd_mul.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTERND_MUL_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTERND_MUL_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatternd_mul : scatternd_op -{ - scatternd_mul() {} - - auto reduction() const - { - return [](auto& x, const auto& y) { x *= y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatternd_none.hpp b/docker/rocm/migraphx/include/migraphx/op/scatternd_none.hpp deleted file mode 100644 index aacdc77d8..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatternd_none.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTERND_NONE_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTERND_NONE_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct scatternd_none : scatternd_op -{ - scatternd_none() {} - - auto reduction() const - { - return [](auto& x, const auto& y) { x = y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/scatternd_op.hpp b/docker/rocm/migraphx/include/migraphx/op/scatternd_op.hpp deleted file mode 100644 index 574dedd86..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/scatternd_op.hpp +++ /dev/null @@ -1,157 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SCATTERND_OP_HPP -#define MIGRAPHX_GUARD_OPERATORS_SCATTERND_OP_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { -/** - * @brief - * N-dimensional Scatter operations. This struct is parent class to ops which differ in what formula - * is used to reduce (combine old and new values of) the scattered value. It was originally based - * on Onnx ScatterND operation (see - * https://github.com/onnx/onnx/blob/main/docs/Operators.md#ScatterND) and is also similar to Numpy - * numpy.add.at(). - * - * @tparam Derived a template parameter in the CRTP inheritance idiom, represents one of the child - * operations. - */ -template -struct scatternd_op : op_name -{ - /** Validate input shapes and return the correct output shape. For Scatter ops, the output - * is the same shape as the data tensor (first input), but cast to a standard shape. - * - */ - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(3); - auto data_shape = inputs.front(); - auto index_shape = inputs.at(1); - auto upd_shape = inputs.back(); - - auto r = data_shape.ndim(); - auto q = index_shape.ndim(); - size_t k; - if(index_shape.dynamic()) - { - // the rank of the output is a function of k, so k must be fixed. - if(not index_shape.dyn_dims().back().is_fixed()) - { - MIGRAPHX_THROW( - "GATHERND: last dimension of indices tensor must be fixed (min=max)"); - } - k = index_shape.dyn_dims().back().min; - } - else - k = index_shape.lens().back(); - - // Checks on the sizes of input tensors - if(q + r != upd_shape.ndim() + k + 1) - MIGRAPHX_THROW("ScatterND: ranks of inputs don't match. " + std::to_string(q) + " + " + - std::to_string(r) + " - " + std::to_string(k) + - " - 1 != " + std::to_string(upd_shape.ndim())); - if(k > r) - MIGRAPHX_THROW("ScatterND: index of size " + std::to_string(k) + - " is too large for tensor of rank " + std::to_string(r)); - - // Convert all static shape dimensions to dynamic so they can be compared. - // It's possible for some of the 3 inputs to be dynamic shapes and some static, - // but any dynamic dimension that's compared to a static dimension must be fixed. - auto ind_dims = index_shape.to_dynamic().dyn_dims(); - auto upd_dims = upd_shape.to_dynamic().dyn_dims(); - auto data_dims = data_shape.to_dynamic().dyn_dims(); - - // Check that corresponding portions of tensor shapes match. - // Brackets around q - 1 are placed for safeguarding against the breaking iterator out of - // vector range. - if(not(std::equal(ind_dims.begin(), ind_dims.begin() + (q - 1), upd_dims.begin()) and - std::equal(data_dims.begin() + k, data_dims.end(), upd_dims.begin() + (q - 1)))) - MIGRAPHX_THROW("ScatterND: incorrect update shape. Update dimensions must match " - "indices and data."); - - if(data_shape.dynamic()) - return data_shape; - else if(data_shape.broadcasted()) - { - return {data_shape.type(), data_shape.lens()}; - } - else - { - return data_shape.with_lens(data_shape.lens()); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - auto& self = static_cast(*this); - visit_all(result, args[0], args[2])([&](auto output, auto data, auto updates) { - std::copy(data.begin(), data.end(), output.begin()); - args[1].visit([&](auto indices) { - auto updates_shape = updates.get_shape(); - auto updates_std = shape{updates_shape.type(), updates_shape.lens()}; - auto indices_shape = indices.get_shape(); - auto k = indices_shape.lens().back(); - auto q = indices_shape.ndim(); - auto r = dyn_out.computed_shape.ndim(); - for(auto i = 0u; i < updates_shape.elements(); ++i) - { - auto updates_idx = updates_std.multi(i); - std::vector indices_idx(q, 0); - std::copy( - updates_idx.begin(), updates_idx.begin() + (q - 1), indices_idx.begin()); - auto index_start = indices.begin() + - indices_shape.index(indices_idx.begin(), indices_idx.end()); - auto index_end = index_start + k; - - std::vector out_idx(r, 0); - std::copy(index_start, index_end, out_idx.begin()); - std::copy( - updates_idx.begin() + (q - 1), updates_idx.end(), out_idx.begin() + k); - - self.reduction()(output[dyn_out.computed_shape.index(out_idx)], updates[i]); - } - }); - }); - - return result; - } - - auto init() const {} - scatternd_op() {} -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/select_module.hpp b/docker/rocm/migraphx/include/migraphx/op/select_module.hpp deleted file mode 100644 index cf0985379..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/select_module.hpp +++ /dev/null @@ -1,150 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SELECT_MODULE_HPP -#define MIGRAPHX_GUARD_OPERATORS_SELECT_MODULE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct select_module -{ - shape output_dyn_shapes; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.output_dyn_shapes, "output_dyn_shapes")); - } - - std::string name() const { return "select_module"; } - - shape compute_shape(const std::vector& inputs, const std::vector&) const - { - check_shapes{inputs, *this, true}.has_at_least(1); - return shape{output_dyn_shapes}; - } - - std::vector get_input_parameter_names(module_ref mod) const - { - auto param_names = mod->get_parameter_names(); - std::vector ret; - std::copy_if(param_names.cbegin(), - param_names.cend(), - std::back_inserter(ret), - [](auto pn) { return not contains(pn, "#output_"); }); - std::sort(ret.begin(), ret.end()); - return ret; - } - - std::vector get_output_parameter_names(module_ref mod) const - { - auto param_names = mod->get_parameter_names(); - std::vector ret; - std::copy_if(param_names.cbegin(), - param_names.cend(), - std::back_inserter(ret), - [](auto pn) { return contains(pn, "#output_"); }); - // needs to be sorted to ensure output parameter ordering - std::sort(ret.begin(), ret.end()); - return ret; - } - - argument compute(const shape&, - const std::vector& args, - const std::vector& submodule_list, - const std::function( - module_ref&, const std::unordered_map&)>& run) const - { - // Find submodule with input parameter shapes exactly the same as the input instruction - // arguments. Assuming instruction arguments are in the same order as the instruction - // parameters. - auto module_iter = - std::find_if(submodule_list.cbegin(), submodule_list.cend(), [&](module_ref mr) { - auto in_param_names = get_input_parameter_names(mr); - auto param_shapes = mr->get_parameter_shapes(); - assert(in_param_names.size() <= args.size()); - return std::equal( - in_param_names.cbegin(), - in_param_names.cend(), - args.cbegin(), - [&](auto p_name, auto a) { return a.get_shape() == param_shapes[p_name]; }); - }); - - if(module_iter == submodule_list.end()) - { - MIGRAPHX_THROW("SELECT_MODULE: no compatible submodules found for given input shapes"); - } - - auto* module_to_run = *module_iter; - std::unordered_map p_map; - - // add input parameters to parameter_map - auto in_param_names = get_input_parameter_names(module_to_run); - assert(in_param_names.size() <= args.size()); - std::transform(in_param_names.begin(), - in_param_names.end(), - args.begin(), - std::inserter(p_map, p_map.end()), - [&](auto&& name, auto&& a) { return std::make_pair(name, a); }); - - // One tuple output parameter in main module to multiple output parameters in submodule - auto out_param_names = get_output_parameter_names(module_to_run); - auto param_shapes = module_to_run->get_parameter_shapes(); - auto output_sub_objects = args.back().get_sub_objects(); - assert(out_param_names.size() == output_sub_objects.size()); - std::transform(out_param_names.begin(), - out_param_names.end(), - output_sub_objects.begin(), - std::inserter(p_map, p_map.end()), - [&](auto&& name, auto&& a) { - auto ps = param_shapes.at(name); - if(a.get_shape() != ps) - { - assert(ps.bytes() <= a.get_shape().bytes()); - return std::make_pair(name, a.reshape(ps)); - } - else - { - return std::make_pair(name, a); - } - }); - auto results = run(module_to_run, p_map); - return argument{results}; - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sigmoid.hpp b/docker/rocm/migraphx/include/migraphx/op/sigmoid.hpp deleted file mode 100644 index 37e23b0b5..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sigmoid.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SIGMOID_HPP -#define MIGRAPHX_GUARD_OPERATORS_SIGMOID_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sigmoid : unary -{ - std::string point_op() const { return "1.f / (1.f + ${function:exp}(-${0}))"; } - auto apply() const - { - return [](auto x) { return 1.f / (1.f + std::exp(-x)); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sign.hpp b/docker/rocm/migraphx/include/migraphx/op/sign.hpp deleted file mode 100644 index 813d1f5b0..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sign.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SIGN_HPP -#define MIGRAPHX_GUARD_OPERATORS_SIGN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sign : unary -{ - std::string point_op() const { return "(${0} > 0 ? 1 : ((${0} < 0) ? -1 : 0))"; } - auto apply() const - { - return [](auto x) { return (x > 0 ? 1 : ((x < 0) ? -1 : 0)); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sin.hpp b/docker/rocm/migraphx/include/migraphx/op/sin.hpp deleted file mode 100644 index 6f9e4c663..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sin.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SIN_HPP -#define MIGRAPHX_GUARD_OPERATORS_SIN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sin : unary -{ - auto apply() const - { - return [](auto x) { return std::sin(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sinh.hpp b/docker/rocm/migraphx/include/migraphx/op/sinh.hpp deleted file mode 100644 index 1fdeabeab..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sinh.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SINH_HPP -#define MIGRAPHX_GUARD_OPERATORS_SINH_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sinh : unary -{ - auto apply() const - { - return [](auto x) { return std::sinh(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/slice.hpp b/docker/rocm/migraphx/include/migraphx/op/slice.hpp deleted file mode 100644 index 9e3df9ce3..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/slice.hpp +++ /dev/null @@ -1,511 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SLICE_HPP -#define MIGRAPHX_GUARD_OPERATORS_SLICE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Slice operator that accepts variable axes, starts and ends. - * All of `starts`, `ends`, and `axes` must be supplied by either - * their attribute or an input (but not both). - * - * Valid calls: - * slice(input); axes, starts, ends set - * slice(input, starts); axes, ends set - * slice(input, ends); starts, axes set - * slice(input, axes); starts, ends set - * slice(input, starts, ends); axes set - * slice(input, starts, axes); ends set - * slice(input, ends, axes); starts set - * slice(input, start, ends, axes); none set - * - * Attributes: - * axes: constant axes to slice over (optional) - * starts: constant slice starting indices (optional) - * ends: constant slice ending indices (optional) - * - * Parameters: - * data: the input tensor to slice (dynamic or static shape) - * input_starts: starting indices of slice (optional, static shape) - * input_ends: ending indices of slice (optional, static shape) - * input_axes: axes to slice over (optional, static shape) - */ -struct slice -{ - std::vector axes{}; - std::vector starts{}; - std::vector ends{}; - - /** - * Named arrays for the set attribute possibilities. - */ - static constexpr std::array all_set = {true, true, true}; - static constexpr std::array ends_axes = {false, true, true}; - static constexpr std::array starts_axes = {true, false, true}; - static constexpr std::array starts_ends = {true, true, false}; - static constexpr std::array axes_only = {false, false, true}; - static constexpr std::array ends_only = {false, true, false}; - static constexpr std::array starts_only = {true, false, false}; - static constexpr std::array none_set = {false, false, false}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes"), f(self.starts, "starts"), f(self.ends, "ends")); - } - - /** - * Ensure that attribute axes is within limits. - * Will attempt to normalize starts and ends; but will use the dynamic_dimension.max - * values for dynamic shapes. This makes it so you have to renormalize for - * non-fixed dynamic_dimensions. - */ - value attributes() const - { - value normalize_axes = value::object{}; - normalize_axes["axes"] = value::array{normalize_attribute::include_min}; - normalize_axes["starts"] = value::array{normalize_attribute::clip_max, - normalize_attribute::clip_min, - normalize_attribute::include_max, - normalize_attribute::use_len, - normalize_attribute::include_min}; - normalize_axes["ends"] = value::array{normalize_attribute::clip_max, - normalize_attribute::clip_min, - normalize_attribute::include_max, - normalize_attribute::use_len, - normalize_attribute::include_min}; - return {{"normalize_axes", normalize_axes}}; - } - - std::string name() const { return "slice"; } - - /** - * Computes the slice output shape dimensions for given starts, ends,and axes. - * Templated to also handle tensor views. - * Possibly different type between [in_starts, in_ends] and [in_axes] if in_axes is this - * object's axes attribute. Assumes in_starts and in_ends are normalized; in_axes are valid. - */ - template - std::vector - lens_calc(const std::vector& lengths, A in_starts, A in_ends, B in_axes) const - { - auto new_lens = lengths; - for(std::size_t i = 0; i < in_axes.size(); ++i) - { - auto axis = in_axes[i]; - new_lens[axis] = in_ends[i] - in_starts[i]; - } - return new_lens; - } - - /// Get the attributes that are non-empty - std::array get_set_attributes() const - { - std::array, 3> attrs = {this->starts, this->ends, this->axes}; - std::array bool_vec; - std::transform( - attrs.cbegin(), attrs.cend(), bool_vec.begin(), [](auto a) { return not a.empty(); }); - return bool_vec; - } - - /// Helper function for normalize_compute_shape() - shape compute_two_or_more(std::vector inputs) const - { - auto input_shape = inputs[0]; - auto set_attributes = get_set_attributes(); - // check that inputs [1, end) are all 1D, have the same - // dimension, and are static - check_shapes{inputs.begin() + 1, - inputs.end(), - std::string("SLICE: inputs (starts, ends, and input_axes)"), - false} - .only_dims(1) - .same_dims(); - auto dds = input_shape.to_dynamic().dyn_dims(); - if(inputs.size() == 2) - { - if(set_attributes == ends_axes) - { - // attr ends and axes set; inputs are (data, input_starts) - if(inputs[1].lens().at(0) != axes.size()) - { - MIGRAPHX_THROW("SLICE: 2 input and attributes mismatch"); - } - std::for_each(axes.cbegin(), axes.cend(), [&](const auto& axis) { - dds.at(axis) = {0, dds.at(axis).max}; - }); - } - else if(set_attributes == starts_axes) - { - // attr starts and axes set; inputs are (data, input_ends) - if(inputs[1].lens().at(0) != axes.size()) - { - MIGRAPHX_THROW("SLICE: 2 input and attributes mismatch"); - } - std::for_each(axes.cbegin(), axes.cend(), [&](const auto& axis) { - dds.at(axis) = {0, dds.at(axis).max}; - }); - } - else if(set_attributes == starts_ends) - { - // attr starts and ends set; inputs are (data, input_axes) - if(inputs[1].lens().at(0) != starts.size()) - { - MIGRAPHX_THROW("SLICE: 2 input and attributes mismatch"); - } - std::transform(dds.begin(), dds.end(), dds.begin(), [](auto dd) { - return shape::dynamic_dimension{0, dd.max}; - }); - } - else - { - MIGRAPHX_THROW("SLICE: Invalid 2 input and attributes configuration"); - } - } - else if(inputs.size() == 3) - { - if(set_attributes == axes_only) - { - // attr axes set; inputs are (data, input_starts, input_ends) - if(inputs[1].lens().at(0) != axes.size()) - { - MIGRAPHX_THROW("SLICE: 3 input and attributes mismatch"); - } - std::for_each(axes.cbegin(), axes.cend(), [&](const auto& axis) { - dds.at(axis) = {0, dds.at(axis).max}; - }); - } - else if(set_attributes == ends_only) - { - // attr ends set; inputs are (data, input_starts, input_axes) - if(inputs[1].lens().at(0) != ends.size()) - { - MIGRAPHX_THROW("SLICE: 3 input and attributes mismatch"); - } - std::transform(dds.begin(), dds.end(), dds.begin(), [](auto dd) { - return shape::dynamic_dimension{0, dd.max}; - }); - } - else if(set_attributes == starts_only) - - { - // attr starts set; inputs are (data, input_ends, input_axes) - if(inputs[1].lens().at(0) != starts.size()) - { - MIGRAPHX_THROW("SLICE: 3 input and attributes mismatch"); - } - std::transform(dds.begin(), dds.end(), dds.begin(), [](auto dd) { - return shape::dynamic_dimension{0, dd.max}; - }); - } - else - { - MIGRAPHX_THROW("Invalid 3 input and attributes configuration"); - } - } - else - { - // all 4 inputs (data, inputs_starts, input_ends, input_axes) - std::transform(dds.begin(), dds.end(), dds.begin(), [](auto dd) { - return shape::dynamic_dimension{0, dd.max}; - }); - } - return shape{input_shape.type(), dds}; - } - - // uses the normalize_axes flag to normalize axes, starts, and ends - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1, 2, 3, 4); - if(inputs.size() == 1) - { - auto input_shape = inputs[0]; - auto set_attributes = get_set_attributes(); - if(set_attributes != all_set) - { - MIGRAPHX_THROW("SLICE 1_arg: Invalid 1 input and attributes configuration"); - } - // NOTE: make sure to update how normalization works here if this type of slicing is - // changed to be allowed - if(input_shape.dynamic() and std::any_of(axes.begin(), axes.end(), [&](auto axis) { - return not input_shape.dyn_dims()[axis].is_fixed(); - })) - { - MIGRAPHX_THROW( - "SLICE 1_arg: slicing is not allowed on non-fixed dynamic input axis "); - } - if(input_shape.dynamic()) - { - return shape{ - input_shape.type(), - lens_calc(input_shape.min_lens(), this->starts, this->ends, this->axes), - lens_calc(input_shape.max_lens(), this->starts, this->ends, this->axes), - {}}; - } - else - { - return shape{input_shape.type(), - lens_calc(input_shape.lens(), this->starts, this->ends, this->axes), - input_shape.strides()}; - } - } - else - { - return compute_two_or_more(inputs); - } - } - - /** - * Calculates the starting offset for the sliced tensor. - * Used in compute when only data input and all other information are in the attributes. - * - * \param s static input shape - */ - auto compute_offset(const shape& s) const - { - const std::vector& lens = s.lens(); - const std::vector& strides = s.strides(); - auto offset = 0; - if(not axes.empty()) - { - for(std::size_t i = 0; i < axes.size(); i++) - { - auto axis = axes[i]; - offset += starts[i] * strides[axis]; - } - } - else - { - for(std::size_t axis = 0; axis < lens.size(); axis++) - { - offset += starts[axis] * strides[axis]; - } - } - return offset * s.type_size(); - } - - /** - * Calculates the starting offset for the sliced tensor (for aliasing). - * Used for 2-4 inputs to `slice. - * - * \param s static input shape - * \param input_starts starting indices of slice - * \param ax_vec axes to slice on - */ - template - auto compute_offset(const shape& s, const T& input_starts, const T& ax_vec) const - { - auto ret = 0; - for(std::size_t i = 0; i < ax_vec.size(); ++i) - { - auto axis = ax_vec[i]; - ret += input_starts[i] * s.strides().at(axis); - } - return ret * s.type_size(); - } - - /** - * If given, normalize the inputs. Otherwise get from operator attributes. - * Return the values in a map. - * - * Parameters - * input_shape: static shape of the input - * input_starts: optional - * input_ends: optional - * input_ends: optional - */ - std::unordered_map> - normalize_starts_ends_axes(shape input_shape, - const optional>& input_starts, - const optional>& input_ends, - const optional>& input_axes) const - { - auto axes_attrs = this->attributes().at("normalize_axes"); - std::vector norm_starts; - std::vector norm_ends; - std::vector norm_axes; - if(input_axes) - { - norm_axes = normalize_axes(input_axes.value(), - input_shape, - axes_attrs.at("axes"), - "Slice variable input_axes"); - } - else - { - norm_axes = this->axes; - } - if(input_starts) - { - norm_starts = normalize_indices(input_starts.value(), - norm_axes, - input_shape, - axes_attrs.at("starts"), - "Slice variable input_starts"); - } - else - { - norm_starts = this->starts; - } - if(input_ends) - { - norm_ends = normalize_indices(input_ends.value(), - norm_axes, - input_shape, - axes_attrs.at("ends"), - "Slice variable input ends"); - } - else - { - norm_ends = this->ends; - } - return {{"norm_starts", norm_starts}, {"norm_ends", norm_ends}, {"norm_axes", norm_axes}}; - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - auto input = args[0]; - auto input_shape = input.get_shape(); - if(args.size() == 1) - { - std::size_t offset = compute_offset(input_shape); - return {dyn_out.computed_shape, [=] { return input.data() + offset; }}; - } - else - { - // Note that we re-normalize both the attributes and inputs because of the non-fixed - // dynamic input shape case. It's possible to only re-normalize if slicing over - // non-fixed dynamic_dimensions. - auto set_attributes = get_set_attributes(); - std::unordered_map> norm_inputs; - if(set_attributes == ends_axes) - { - // attr ends and axes set; inputs are (data, input_starts) - args[1].visit([&](auto input_starts) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - input_starts.template to_vector(), - this->ends, - this->axes); - }); - } - else if(set_attributes == starts_axes) - { - // attr starts and axes set; inputs are (data, input_ends) - args[1].visit([&](auto input_ends) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - this->starts, - input_ends.template to_vector(), - this->axes); - }); - } - else if(set_attributes == starts_ends) - { - // attr starts and ends set; inputs are (data, input_axes) - args[1].visit([&](auto input_axes) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - this->starts, - this->ends, - input_axes.template to_vector()); - }); - } - else if(set_attributes == axes_only) - { - // attr axes set; inputs are (data, input_starts, input_ends) - visit_all(args[1], args[2])([&](auto input_starts, auto input_ends) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - input_starts.template to_vector(), - input_ends.template to_vector(), - this->axes); - }); - } - else if(set_attributes == ends_only) - { - // attr ends set; inputs are (data, input_starts, input_axes) - visit_all(args[1], args[2])([&](auto input_starts, auto input_axes) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - input_starts.template to_vector(), - this->ends, - input_axes.template to_vector()); - }); - } - else if(set_attributes == starts_only) - { - // attr starts set; inputs are (data, input_ends, input_axes) - visit_all(args[1], args[2])([&](auto input_ends, auto input_axes) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - this->starts, - input_ends.template to_vector(), - input_axes.template to_vector()); - }); - } - else - { - // no attr set, all inputs - visit_all(args[1], args[2], args[3])( - [&](auto input_starts, auto input_ends, auto input_axes) { - norm_inputs = - normalize_starts_ends_axes(input_shape, - input_starts.template to_vector(), - input_ends.template to_vector(), - input_axes.template to_vector()); - }); - } - auto offset = compute_offset( - input_shape, norm_inputs.at("norm_starts"), norm_inputs.at("norm_axes")); - shape calc_shape = shape{input_shape.type(), - lens_calc(input_shape.lens(), - norm_inputs.at("norm_starts"), - norm_inputs.at("norm_ends"), - norm_inputs.at("norm_axes")), - input_shape.strides()}; - return {calc_shape, [=] { return input.data() + offset; }}; - } - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/softmax.hpp b/docker/rocm/migraphx/include/migraphx/op/softmax.hpp deleted file mode 100644 index fd4683681..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/softmax.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SOFTMAX_HPP -#define MIGRAPHX_GUARD_OPERATORS_SOFTMAX_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct softmax -{ - int64_t axis = 1; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "softmax"; } - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto s0 = inputs[0]; - if(s0.dynamic() or s0.packed()) - { - return s0; - } - else - { - return {s0.type(), s0.lens()}; - } - } - - auto output() const - { - return [=](auto x, auto y) { return x / y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sqdiff.hpp b/docker/rocm/migraphx/include/migraphx/op/sqdiff.hpp deleted file mode 100644 index 82f0b1557..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sqdiff.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SQDIFF_HPP -#define MIGRAPHX_GUARD_OPERATORS_SQDIFF_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sqdiff : binary -{ - std::string point_op() const { return "(${0} - ${1}) * (${0} - ${1})"; } - auto apply() const - { - return [](auto x, auto y) { return (x - y) * (x - y); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sqrt.hpp b/docker/rocm/migraphx/include/migraphx/op/sqrt.hpp deleted file mode 100644 index c0a8c9812..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sqrt.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SQRT_HPP -#define MIGRAPHX_GUARD_OPERATORS_SQRT_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sqrt : unary -{ - auto apply() const - { - return [](auto x) { return std::sqrt(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/squeeze.hpp b/docker/rocm/migraphx/include/migraphx/op/squeeze.hpp deleted file mode 100644 index a237a0a58..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/squeeze.hpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SQUEEZE_HPP -#define MIGRAPHX_GUARD_OPERATORS_SQUEEZE_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct squeeze -{ - std::vector axes; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes")); - } - - value attributes() const - { - value normalize; - normalize["axes"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "squeeze"; } - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto input_shape = inputs[0]; - if(input_shape.dynamic()) - { - // Allow for any dynamic_dimension that intersects with {1, 1}. - // Assuming that the shape at run-time will be compatible. - if(std::any_of(axes.begin(), axes.end(), [&](auto axis) { - return not input_shape.dyn_dims() - .at(axis) - .intersection(shape::dynamic_dimension{1, 1}) - .has_value(); - ; - })) - { - MIGRAPHX_THROW( - "SQUEEZE: dynamic axis dimension should have an intersection with {1, 1}"); - } - std::vector dyn_dims = {}; - if(axes.empty()) - { - std::copy_if(input_shape.dyn_dims().cbegin(), - input_shape.dyn_dims().cend(), - std::back_inserter(dyn_dims), - [&](auto dd) { return dd != 1; }); - } - else - { - for(auto i : range(input_shape.ndim())) - { - if(std::find(axes.begin(), axes.end(), i) == axes.end()) - { - dyn_dims.push_back(input_shape.dyn_dims()[i]); - } - } - } - return {input_shape.type(), dyn_dims}; - } - else - { - auto type = input_shape.type(); - auto old_lens = input_shape.lens(); - auto old_strides = input_shape.strides(); - if(std::any_of( - axes.begin(), axes.end(), [&](auto axis) { return old_lens[axis] != 1; })) - { - MIGRAPHX_THROW("SQUEEZE: static axis dimension should be equal to 1"); - } - std::vector new_lens; - std::vector new_strides; - if(axes.empty()) - { - for(auto i : range(old_lens.size())) - { - if(old_lens[i] != 1) - { - new_lens.push_back(old_lens[i]); - new_strides.push_back(old_strides[i]); - } - } - } - else - { - for(auto i : range(old_lens.size())) - { - if(std::find(axes.begin(), axes.end(), i) == axes.end()) - { - new_lens.push_back(old_lens[i]); - new_strides.push_back(old_strides[i]); - } - } - } - if(new_lens.empty()) - { - return shape{type}; - } - else - { - return shape{type, new_lens, new_strides}; - } - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/step.hpp b/docker/rocm/migraphx/include/migraphx/op/step.hpp deleted file mode 100644 index 3a24fcdce..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/step.hpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_STEP_HPP -#define MIGRAPHX_GUARD_OPERATORS_STEP_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct step -{ - std::vector axes; - std::vector steps; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes"), f(self.steps, "steps")); - } - - value attributes() const - { - value normalize; - normalize["axes"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "step"; } - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1); - auto input = inputs.at(0); - auto in_lens = input.lens(); - auto t = input.type(); - - if(axes.size() != steps.size()) - { - MIGRAPHX_THROW("STEP: attribute axes {" + to_string_range(axes) + - "} has different dimensions from step {" + to_string_range(steps) + - "}."); - } - - if(std::any_of(axes.begin(), axes.end(), [&](auto axis) { return axis >= in_lens.size(); })) - { - MIGRAPHX_THROW("STEP: axis value is out of range!"); - } - - auto lens = in_lens; - auto strides = input.strides(); - for(auto i : range(axes.size())) - { - auto axis = axes[i]; - auto step = steps[i]; - lens[axis] = (in_lens[axis] + step - 1) / step; - strides[axis] *= step; - } - - return {t, lens, strides}; - } - - argument compute(shape output_shape, std::vector args) const - { - return args[0].reshape(output_shape); - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/sub.hpp b/docker/rocm/migraphx/include/migraphx/op/sub.hpp deleted file mode 100644 index 28c0cead1..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/sub.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_SUB_HPP -#define MIGRAPHX_GUARD_OPERATORS_SUB_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct sub : binary -{ - std::string point_function() const { return "-"; } - auto apply() const - { - return [](auto x, auto y) { return x - y; }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/tan.hpp b/docker/rocm/migraphx/include/migraphx/op/tan.hpp deleted file mode 100644 index 50b2067f0..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/tan.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_TAN_HPP -#define MIGRAPHX_GUARD_OPERATORS_TAN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct tan : unary -{ - auto apply() const - { - return [](auto x) { return std::tan(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/tanh.hpp b/docker/rocm/migraphx/include/migraphx/op/tanh.hpp deleted file mode 100644 index 367713258..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/tanh.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_TANH_HPP -#define MIGRAPHX_GUARD_OPERATORS_TANH_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct tanh : unary -{ - auto apply() const - { - return [](auto x) { return std::tanh(x); }; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/topk.hpp b/docker/rocm/migraphx/include/migraphx/op/topk.hpp deleted file mode 100644 index 863119b82..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/topk.hpp +++ /dev/null @@ -1,166 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_GATHER_HPP -#define MIGRAPHX_GUARD_OPERATORS_GATHER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct topk -{ - int64_t k = 1; - int64_t axis = 0; - bool largest = true; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.k, "k"), f(self.axis, "axis"), f(self.largest, "largest")); - } - - value attributes() const - { - value normalize; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "topk"; } - - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1).standard(); - auto lens = inputs.at(0).lens(); - auto type = inputs.at(0).type(); - - lens[axis] = k; - - shape s_val{type, lens}; - shape s_ind{shape::int64_type, lens}; - - return shape({s_val, s_ind}); - } - - template - struct heap_vector - { - std::vector data; - Compare compare; - - heap_vector(const std::vector& val, Compare comp) : data(val), compare(std::move(comp)) - { - std::make_heap(data.begin(), data.end(), compare); - } - - void try_push(T val) - { - if(not compare(val, data.front())) - return; - - std::pop_heap(data.begin(), data.end(), compare); - data.back() = val; - std::push_heap(data.begin(), data.end(), compare); - } - - std::vector sort() - { - auto sorted_data = data; - std::sort_heap(sorted_data.begin(), sorted_data.end(), compare); - return sorted_data; - } - }; - - template - heap_vector make_heap(std::vector val, Compare compare) const - { - return {std::move(val), std::move(compare)}; - } - - argument compute(const shape& output_shape, std::vector args) const - { - auto vec_ss = output_shape.sub_shapes(); - argument res_val{vec_ss.front()}; - argument res_ind{vec_ss.back()}; - auto in_s = args.front().get_shape(); - auto out_s = vec_ss.front(); - auto comp_lens = in_s.lens(); - auto axis_dim = comp_lens[axis]; - - // compute shape - comp_lens[axis] = 1; - shape comp_s{in_s.type(), comp_lens}; - visit_all(res_val, args.front())([&](auto out_val, auto input) { - auto* out_ind = res_ind.cast(); - par_for(comp_s.elements(), [&](auto i) { - auto idx = comp_s.multi(i); - std::vector indices(k); - std::iota(indices.begin(), indices.end(), 0); - - auto comp = [&](auto i1, auto i2) { - auto idx1 = idx; - auto idx2 = idx; - idx1[axis] = i1; - idx2[axis] = i2; - return this->largest - ? std::greater<>{}(input[in_s.index(idx1)], input[in_s.index(idx2)]) - : std::less<>{}(input[in_s.index(idx1)], input[in_s.index(idx2)]); - }; - - auto hp = this->make_heap(indices, comp); - for(std::size_t ii = indices.size(); ii < axis_dim; ++ii) - { - hp.try_push(ii); - } - auto sorted_indices = hp.sort(); - auto out_idx = idx; - auto in_idx = idx; - for(auto j : range(sorted_indices.size())) - { - out_idx[axis] = j; - in_idx[axis] = sorted_indices[j]; - out_val[out_s.index(out_idx)] = input[in_s.index(in_idx)]; - out_ind[out_s.index(out_idx)] = sorted_indices[j]; - } - }); - }); - - return {{res_val, res_ind}}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/transpose.hpp b/docker/rocm/migraphx/include/migraphx/op/transpose.hpp deleted file mode 100644 index 1b8c1b5af..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/transpose.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_TRANSPOSE_HPP -#define MIGRAPHX_GUARD_OPERATORS_TRANSPOSE_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct transpose -{ - std::vector dims; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.dims, "permutation")); - } - - std::string name() const { return "transpose"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto input = inputs.at(0); - - if(dims.size() != input.ndim()) - { - MIGRAPHX_THROW("TRANSPOSE: Permutation has wrong number of axes"); - } - std::vector axes(dims.size()); - std::iota(axes.begin(), axes.end(), 0); - if(not std::is_permutation(axes.begin(), axes.end(), dims.begin())) - { - MIGRAPHX_THROW("TRANSPOSE: Invalid permutation"); - } - - if(input.dynamic()) - { - std::vector output_dyn_dims(input.ndim()); - std::transform(dims.cbegin(), dims.cend(), output_dyn_dims.begin(), [&](auto dim) { - return input.dyn_dims()[dim]; - }); - return {input.type(), output_dyn_dims}; - } - else - { - auto input_lens = input.lens(); - auto input_strides = input.strides(); - - std::vector output_lens(input.ndim()); - std::vector output_strides(input.ndim()); - for(std::size_t i = 0; i < input.ndim(); i++) - { - output_lens[i] = input_lens[dims[i]]; - output_strides[i] = input_strides[dims[i]]; - } - return {input.type(), output_lens, output_strides}; - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/unary.hpp b/docker/rocm/migraphx/include/migraphx/op/unary.hpp deleted file mode 100644 index 83d5a1a71..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/unary.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_UNARY_HPP -#define MIGRAPHX_GUARD_OPERATORS_UNARY_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -template -struct unary : op_name -{ - std::string point_function() const { return this->name(); } - std::string point_op() const - { - const auto& self = static_cast(*this); - auto pf = self.point_function(); - if(pf.empty()) - return {}; - if(with_char(::ispunct)(pf.front())) - { - return pf + "${0}"; - } - else - { - return "${function:" + pf + "}(${0})"; - } - } - value base_attributes() const - { - const auto& self = static_cast(*this); - return {{"pointwise", true}, {"point_op", self.point_op()}}; - } - value attributes() const { return base_attributes(); } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, static_cast(*this), true}.has(1); - auto s = inputs.at(0); - if(s.dynamic() or s.scalar()) - { - return s; - } - else if(s.broadcasted()) - { - return {s.type(), s.lens()}; - } - else - { - return s.with_lens(s.lens()); - } - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - result.visit([&](auto output) { - args[0].visit([&](auto input) { - par_transform(input.begin(), - input.end(), - output.begin(), - static_cast(*this).apply()); - }); - }); - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/unary_not.hpp b/docker/rocm/migraphx/include/migraphx/op/unary_not.hpp deleted file mode 100644 index 01d8364a9..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/unary_not.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_UNARY_NOT_HPP -#define MIGRAPHX_GUARD_OPERATORS_UNARY_NOT_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct unary_not : unary -{ - std::string point_function() const { return "!"; } - auto apply() const - { - return [](auto x) { return not x; }; - } - - std::string name() const { return "not"; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/undefined.hpp b/docker/rocm/migraphx/include/migraphx/op/undefined.hpp deleted file mode 100644 index d685d3bf8..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/undefined.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_UNDEFINED_HPP -#define MIGRAPHX_GUARD_RTGLIB_UNDEFINED_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct undefined -{ - std::string name() const { return "undefined"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return {}; - } - - argument compute(const shape&, const std::vector&) const { return {{}, nullptr}; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/unique.hpp b/docker/rocm/migraphx/include/migraphx/op/unique.hpp deleted file mode 100644 index 67afbbf09..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/unique.hpp +++ /dev/null @@ -1,323 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_OPERATORS_UNIQUE_HPP -#define MIGRAPHX_GUARD_OPERATORS_UNIQUE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -// https://onnx.ai/onnx/operators/onnx__Unique.html -// The Onnx spec refers to numpy specification, used as a reference: -// https://numpy.org/doc/stable/reference/generated/numpy.unique.html - -// Input : Given an array of elements : X. - -// Output(s) : -// 1. Find the unique elements (Y) of input (X). -// -// There are three outputs in addition to the unique elements in Y: -// 2. the indices of the input array that give the unique values -// 3. the indices of the unique array that reconstruct the input array -// 4. the number of times each unique value comes up in the input array - -// Optional Attribute: 'Sorted' = 1 for sorted; = 0 for unsorted. -// Onnx specification makes 'sorted' a default, while Numpy always sorts. -// -// Optional Attribute: 'Axis' is 'None' (default) or a valid int < rank(X). -// Negative values are allowed. -// -// Numpy has the following important note on Axis: -// ------------------------------------------------------------------ -// When an axis is specified the subarrays indexed by the axis are -// sorted. This is done by making the specified axis the first -// dimension of the array (move the axis to the first dimension to -// keep the order of the other axes) and then flattening the subarrays -// in C order. The flattened subarrays are then viewed as a structured -// type with each element given a label, with the effect that we end -// up with a 1-D array of structured types that can be treated in the -// same way as any other 1-D array. The result is that the flattened -// subarrays are sorted in lexicographic order starting with the first -// element. -// ------------------------------------------------------------------ - -struct unique -{ - - template - auto make_idx_less_fn(const T& data, size_t chunk_sz) const - { - return [&data, chunk_sz](auto idx1, auto idx2) { - return std::lexicographical_compare(data.begin() + idx1, - data.begin() + idx1 + chunk_sz, - data.begin() + idx2, - data.begin() + idx2 + chunk_sz); - }; - } - - // CASE SORTED: - // - // To process into a sorted unique series of elements/chunks: - // Chunk size == 1 means a simple element; >1 means a flat representation. - // Steps: first go through the input elements/chunks for uniqueness. - // At the end of this processing, per the sorted sequence of unique elements: - // update/create data structures: y, y_indices, x_rev_indices, y_count - // - // INPUT x: [2, 1, 1, 3, 4, 3], attr_sorted = 1; - - // OUTPUT(s): indices.. - // y_indices: [1, 0, 3, 4] --- first incidence, in terms of index in sequence x - // x_rev_indices: [1, 0, 0, 2, 3, 2] --- x seen in terms of indices of unique sequence y - // y_count: [2, 1, 2, 1] -- count at each y_index. sum = len(x) - - // NOTE: y [1, 2, 3, 4] --- the unique output is constructed from x[y_indices[...]] - - template - auto sorted_uniq_indices(const T& input_data, size_t chunk_sz) const - { - struct y_info - { - size_t y_idx; - size_t x_idx; - size_t ct = 0; - }; - - auto idx_less_fn = make_idx_less_fn(input_data, chunk_sz); - std::map uniq_val_map(idx_less_fn); - - std::tuple, std::vector, std::vector> rv; - auto& [y_indices, x_rev_indices, y_count] = rv; - - // go through all the elements and find the unique elements.. - size_t count_x = input_data.size(); - for(size_t f_idx = 0, x_idx = 0; f_idx < count_x; f_idx += chunk_sz, x_idx++) - { - y_info entry = {.y_idx = uniq_val_map.size(), .x_idx = x_idx}; - auto [itr, added_new] = uniq_val_map.insert({f_idx, entry}); - itr->second.ct++; - x_rev_indices.push_back(itr->second.y_idx); - } - - std::vector y2x_indices(uniq_val_map.size()); - y_indices.resize(uniq_val_map.size()); - y_count.resize(uniq_val_map.size()); - size_t idx = 0; - // the unique elements are now sorted: - // post-processing for all the return indices. - for(const auto& v : uniq_val_map) - { - y2x_indices[v.second.y_idx] = idx; - y_indices[idx] = v.second.x_idx; - y_count[idx] = v.second.ct; - idx++; - } - // update x_rev_indices as per the sorted order of y_indices - for(auto& i : x_rev_indices) - i = y2x_indices[i]; - - return rv; - } - - // CASE UNSORTED: - // - // To process into an un-sorted unique series of elements/chunks: - // For chunk size = 1 is a simple element, else use a flat representation of a tensor obj - // Go through the input elements/chunks one by one with inline processing of indices.. - - // INPUT x: [2, 1, 1, 3, 4, 3], attr_sorted = 0; - - // OUTPUT(s): indices.. - // y_indices: [0, 1, 3, 4] --- first incidence, in terms of index in sequence x - // x_rev_indices: [0, 1, 1, 2, 3, 2] --- x seen in terms of indices of unique sequence y - // y_count: [1, 2, 2, 1] -- count at each y_index. sum = len(x) - - // NOTE: y [2, 1, 3, 4] --- the unique output is constructed from x[y_indices[...]] - // Output data structures: y_indices, x_rev_indices, y_count are processed inline. - - template - auto unsorted_uniq_indices(const T& input_data, size_t chunk_sz) const - { - auto idx_less_fn = make_idx_less_fn(input_data, chunk_sz); - std::map uniq_val_map(idx_less_fn); - - // rv is used for NVRO below.. - std::tuple, std::vector, std::vector> rv; - auto& [y_indices, x_rev_indices, y_count] = rv; - - // go through all the elements and add the unique elements into the map.. - // inline processing for outputs: y_indices, x_rev_indices, y_count - size_t count_x = input_data.size(); - for(size_t f_idx = 0; f_idx < count_x; f_idx += chunk_sz) - { - auto [itr, added_new] = uniq_val_map.insert({f_idx, y_indices.size()}); - if(added_new) - { - y_count.push_back(0); - y_indices.push_back(x_rev_indices.size()); - } - y_count[itr->second]++; - x_rev_indices.push_back(itr->second); - } - - return rv; - } - - // Axis. Default: none. Range: [-rank, rank-1] - std::optional axis; - - // Sorted, Default: 1= sorted. 0 = unsorted. - bool sorted = true; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis"), f(self.sorted, "sorted")); - } - - std::string name() const { return "unique"; } - - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(1); - - auto& sh_x = inputs[0]; - auto lens_x = sh_x.lens(); - size_t dim_x = sh_x.ndim(); - size_t max_uniq_ct = sh_x.elements(); - std::vector d_out; - - if(axis) - { - int64_t t_axis = migraphx::tune_axis(dim_x, *axis, name()); - if(t_axis != 0) - MIGRAPHX_THROW("Unique: Only supports axis = 0 or None"); - - d_out = sh_x.to_dynamic().dyn_dims(); - // only axis = 0 is supported: - max_uniq_ct = lens_x[0]; - // min = 1 unique element; max = full dimension along axis 0 - d_out[0] = {1, max_uniq_ct}; - } - else - { - d_out.push_back({1, max_uniq_ct}); - } - - shape sh_y = {sh_x.type(), d_out}; - // The three outputted Indices are just 1-D: - shape sh_idx{shape::int64_type, {d_out[0]}}; - - return shape({sh_y, sh_idx, sh_idx, sh_idx}); - } - - argument compute(const dyn_output& dyn_out, std::vector args) const - { - auto sh_x = args.front().get_shape(); - auto lens_x = sh_x.lens(); - shape output_shape = dyn_out.computed_shape; - auto vec_ss = output_shape.sub_shapes(); - auto ct_x = sh_x.elements(); - shape sh_y = {vec_ss[0].type(), {ct_x}}; - shape sh_idx = {vec_ss[1].type(), {ct_x}}; - shape sh_x_idx = {vec_ss[1].type(), {ct_x}}; - - argument res_y{sh_y}; - argument res_y_idx{sh_idx}; - argument res_x_rev_idx{sh_idx}; - argument res_y_ct_idx{sh_idx}; - - std::vector out_y_idx; - std::vector out_x_rev_idx; - std::vector out_y_ct; - - // If axis is not none, for >1D tensors, we have to consider - // then, the uniqueness of chunks of sub-tensors: a subsequence of built-ins.. - // For a built-in type, chunk_sz is of course = 1 - size_t chunk_sz = 1; - if(axis) - chunk_sz = ct_x / lens_x[0]; // axis = 0 is supported. - - visit_all(args.front(), res_y)([&](auto x, auto y_flat) { - using o_type = typename decltype(x)::value_type; - std::vector x_in(x.begin(), x.end()); - - std::tie(out_y_idx, out_x_rev_idx, out_y_ct) = - sorted ? sorted_uniq_indices(x_in, chunk_sz) - : unsorted_uniq_indices(x_in, chunk_sz); - - const auto uniq_ct = out_y_idx.size(); - - // construct y from x[indices] in flattened form - // later we reshape y to the final shape.. - auto y_dst = y_flat.begin(); - for(size_t idx = 0; idx < uniq_ct; idx++) - y_dst = copy_n(x_in.begin() + out_y_idx[idx] * chunk_sz, chunk_sz, y_dst); - - std::vector lens_y; - // if axis is specified: - // the output shape keeps the n-1 dimensions of x - if(axis) - { - lens_y = lens_x; - lens_y[0] = uniq_ct; - } - else - { - lens_y = {uniq_ct}; - } - sh_y = {sh_y.type(), lens_y}; - sh_idx = {sh_idx.type(), {uniq_ct}}; - }); - - visit_all(res_y_idx, res_x_rev_idx, res_y_ct_idx)( - [&](auto y_indices, auto x_rev_indices, auto y_count) { - std::copy(out_y_idx.begin(), out_y_idx.end(), y_indices.begin()); - std::copy(out_x_rev_idx.begin(), out_x_rev_idx.end(), x_rev_indices.begin()); - std::copy(out_y_ct.begin(), out_y_ct.end(), y_count.begin()); - sh_x_idx = {sh_idx.type(), {out_x_rev_idx.size()}}; - }); - - return {{res_y.reshape(sh_y), - res_y_idx.reshape(sh_idx), - res_x_rev_idx.reshape(sh_x_idx), - res_y_ct_idx.reshape(sh_idx)}}; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/unknown.hpp b/docker/rocm/migraphx/include/migraphx/op/unknown.hpp deleted file mode 100644 index fc697f207..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/unknown.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_UNKNOWN_HPP -#define MIGRAPHX_GUARD_RTGLIB_UNKNOWN_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct unknown -{ - std::string op; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op")); - } - std::string name() const { return "unknown:" + op; } - shape compute_shape(std::vector input) const - { - if(input.empty()) - return {}; - else - return input.front(); - } - - friend std::ostream& operator<<(std::ostream& os, const unknown& x) - { - os << x.name(); - return os; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/unpack_int4.hpp b/docker/rocm/migraphx/include/migraphx/op/unpack_int4.hpp deleted file mode 100644 index f8e44654b..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/unpack_int4.hpp +++ /dev/null @@ -1,126 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_UNPACK_INT4_HPP -#define MIGRAPHX_GUARD_OPERATORS_UNPACK_INT4_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { -struct unpack_int4 -{ - int64_t axis = -1; - - std::string name() const { return "unpack_int4"; } - - value attributes() const - { - value normalize = value::object{}; - normalize["axis"] = value::array{normalize_attribute::include_min}; - return {{"normalize_axes", normalize}}; - } - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axis, "axis")); - } - - migraphx::shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.same_dims().has(1); - auto in_shape = inputs.front(); - if(in_shape.type() != migraphx::shape::int8_type and - in_shape.type() != migraphx::shape::uint8_type) - { - MIGRAPHX_THROW("UNPACK_INT4: Only Int8 or Uint8 is supported for unpacking"); - } - auto new_lens = in_shape.lens(); - new_lens[axis] *= 2; - return {in_shape.type(), new_lens}; - } - - argument compute(const shape& output_shape, std::vector args) const - { - auto input = args.front(); - auto in_shape = input.get_shape(); - - argument result{output_shape}; - - visit_all(result, input)([&](auto out, auto inp) { - par_for(in_shape.elements(), [&](auto i) { - using type = typename decltype(out)::value_type; - auto data_idx = in_shape.multi(i); - data_idx[axis] *= 2; - if constexpr(std::is_signed{}) - { - // signed input: [Most significant nibble | Least significant nibble] - int8_t val1 = inp[i]; - int8_t val2 = val1; - - // Step1: move the LSN to MSN: - // However avoid doing a left shift of signed quantity - // due to its possible run time error. - uint8_t u_tmp = static_cast(val1); - u_tmp <<= 4; // NOLINT(hicpp-signed-bitwise) - val1 = static_cast(u_tmp); - - // Step2: the sign bit is copied in a right signed-shift: - val1 >>= 4; // NOLINT(hicpp-signed-bitwise) - out[data_idx] = val1; - - data_idx[axis] += 1; - val2 >>= 4; // NOLINT(hicpp-signed-bitwise) - out[data_idx] = val2; - } - else - { - // unpacking of 2 unsigned nibbles: - uint8_t val = inp[i]; - out[data_idx] = val & 0xf; // NOLINT(hicpp-signed-bitwise) - - data_idx[axis] += 1; - out[data_idx] = val >> 4; // NOLINT(hicpp-signed-bitwise) - } - }); - }); - return result; - } -}; -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/unsqueeze.hpp b/docker/rocm/migraphx/include/migraphx/op/unsqueeze.hpp deleted file mode 100644 index 54896331f..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/unsqueeze.hpp +++ /dev/null @@ -1,158 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_UNSQUEEZE_HPP -#define MIGRAPHX_GUARD_OPERATORS_UNSQUEEZE_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -/** - * Adds dimensions to a tensor based on the axes attribute. - * `axes` are based on the number of output shape dimensions and should not contain duplicates. - * `steps` are for modifying dimensions added to the middle of the original shape. - * Each step must be a factor of the original dimension. - * ex: unsqueeze(shape = [3, 4, 10], axes = [2, 4, 5], steps = [2]) -> shape = [3, 4, 2, 5, 1, 1] - * Dynamic shape version does not handle `steps`. - */ -struct unsqueeze -{ - std::vector axes; - std::vector steps; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes"), f(self.steps, "steps")); - } - - value attributes() const - { - value normalize; - normalize["axes"] = - value::array{normalize_attribute::include_min, normalize_attribute::use_output}; - return {{"normalize_axes", normalize}}; - } - - std::string name() const { return "unsqueeze"; } - shape normalize_compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1); - auto input_shape = inputs[0]; - - if(input_shape.dynamic()) - { - if(not steps.empty()) - { - MIGRAPHX_THROW("UNSQUEEZE_dyn: nonempty steps attribute"); - } - std::vector dyn_dims = {}; - auto new_ndim = input_shape.ndim() + axes.size(); - std::size_t k = 0; - for(auto i : range(new_ndim)) - { - if(std::find(axes.begin(), axes.end(), i) != axes.end()) - { - dyn_dims.push_back({1, 1}); - } - else - { - dyn_dims.push_back(input_shape.dyn_dims().at(k++)); - } - } - return {input_shape.type(), dyn_dims}; - } - else - { - auto type = input_shape.type(); - auto old_lens = input_shape.lens(); - auto old_strides = input_shape.strides(); - auto is_scalar = input_shape.scalar(); - - if(is_scalar and old_lens.size() == 1 and old_lens.front() == 1) - return shape{type, old_lens}; - - if(steps.size() > axes.size()) - MIGRAPHX_THROW("UNSQUEEZE: Steps provided with no axis"); - - std::size_t new_size = old_lens.size() + axes.size(); - - std::vector new_lens(new_size); - std::vector new_strides(new_size); - std::size_t p = 0; - for(auto i : range(new_size)) - { - auto axis_idx = std::find(axes.begin(), axes.end(), i) - axes.begin(); - if(axis_idx < axes.size()) - { - std::int64_t step = 1; - if(axis_idx < steps.size()) - step = steps[axis_idx]; - if(step == 0) - MIGRAPHX_THROW("UNSQUEEZE: step must be non-zero"); - if(is_scalar and step != 1) - MIGRAPHX_THROW("UNSQUEEZE: step must be 1 when input is scalar"); - new_lens[i] = step; - if(p < old_strides.size()) - { - if((old_lens[p] % step) != 0) - MIGRAPHX_THROW("UNSQUEEZE: Axis dimension is not divisible by step"); - old_lens[p] /= step; - new_strides[i] = is_scalar ? 1 : old_strides[p] * old_lens[p]; - } - else - { - if(step != 1) - MIGRAPHX_THROW("UNSQUEEZE: Step must be 1 for extra axes"); - new_strides[i] = 1; - } - } - else - { - new_lens[i] = old_lens[p]; - new_strides[i] = old_strides[p++]; - } - } - return shape{type, new_lens, new_strides}; - } - } - argument compute(const dyn_output& dyn_out, std::vector args) const - { - return args[0].reshape(dyn_out.computed_shape); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/op/where.hpp b/docker/rocm/migraphx/include/migraphx/op/where.hpp deleted file mode 100644 index d0f94c312..000000000 --- a/docker/rocm/migraphx/include/migraphx/op/where.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_WHERE_HPP -#define MIGRAPHX_GUARD_OPERATORS_WHERE_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -struct where -{ - std::string name() const { return "where"; } - - value attributes() const { return {{"pointwise", true}, {"point_op", "${0} ? ${1} : ${2}"}}; } - - shape compute_shape(std::vector inputs) const - { - check_shapes shape_checker{inputs, *this, true}; - shape_checker.has(3); - if(auto s = inputs[0]; not s.dynamic() and s.elements() == 1) - check_shapes{std::next(inputs.begin()), inputs.end(), *this, true}.same_dims(); - else - shape_checker.same_dims(); - - auto s1 = inputs.at(1); - auto s2 = inputs.at(2); - if(s1.dynamic() or s2.dynamic()) - { - if(s1 == s2) - return s1; - MIGRAPHX_THROW("WHERE: dynamic input shapes must be the same"); - } - - // Compare two static shapes, returning a standard shape - if(s1 == s2 and s1.packed()) - { - return s1; - } - else if(s1.packed() != s2.packed()) - { - return s1.packed() ? s1 : s2; - } - else if(s1.broadcasted() != s2.broadcasted()) - { - return s1.broadcasted() ? s2.with_lens(s1.lens()) : s1.with_lens(s1.lens()); - } - else - { - return {s1.type(), s1.lens()}; - } - } - - argument compute(shape output_shape, std::vector args) const - { - if(auto s = args[0].get_shape(); not s.dynamic() and s.elements() == 1) - return args[args[0].at() ? 1 : 2].copy(); - - if(output_shape.dynamic()) - output_shape = compute_shape(to_shapes(args)); - argument result{output_shape}; - - visit_all(result, args[1], args[2])([&](auto output, const auto x, const auto y) { - args[0].visit([&](const auto condition) { - par_for(output_shape.elements(), - [&](auto i) { output[i] = condition[i] ? x[i] : y[i]; }); - }); - }); - - return result; - } -}; - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/operation.hpp b/docker/rocm/migraphx/include/migraphx/operation.hpp deleted file mode 100644 index af7c4d5ee..000000000 --- a/docker/rocm/migraphx/include/migraphx/operation.hpp +++ /dev/null @@ -1,1501 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_OPERAND_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_OPERAND_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct context; - -#ifdef DOXYGEN - -/// The operation interface represents an action an instruction will perform. All -/// operation classes must be CopyConstructible. -struct operation -{ - /// A unique name identifying the operation - std::string name() const; - /// An optional method that can be used to finalize the operator before running - void finalize(context& ctx); - /// This is used to compute the resulting shape from an operation. If an - /// operation cannot be run with input shapes, then it should throw an - /// exception. - shape compute_shape(const std::vector& input) const; - /** - * @brief This performs the operation's computation. - * - * This method can be optional when the operation is only used as a placeholder to be lowered - * later on. - * - * @param ctx This is the context created by the `target` during compilation. Implementations - * can use the target's `context` class rather than the `context` interface class. - * @param output Equivalent to running `compute_shape` with each `shape` of the `argument`. - * For a fixed shape, the returned argument will have the same shape as `output`. - * For a dynamic shape, the returned `argument` will be a fixed shape within the bounds - * set in the dynamic shape `output`. - * @param input This is the `argument` result from the previous instruction's computation. - * @return Return an `argument` of the result computation. The `shape` of `argument` should be - * the same the `output` shape. - */ - argument compute(context& ctx, const shape& output, const std::vector& input) const; - /// An optional method to return which argument the output will alias. If - /// there is no aliased output then -1 can be returned. - std::ptrdiff_t output_alias(const std::vector& input) const; - /// An optional stream operator to print the operation. When this is not - /// implemented, it will just print the operation's name. - friend std::ostream& operator<<(std::ostream& os, const operation& op); -}; - -/// Returns true if operation does not require a context to run compute -bool is_context_free(const operation& x); -/// Returns true if operation needs normalization before running compute -bool need_normalization(const operation& x); -/// Returns true if the operation has a finalize method -bool has_finalize(const operation& x); - -#else - -namespace detail { - -namespace operation_operators { - -template -auto operator<<(std::ostream& os, const T& x) -> decltype(os << x.name()) -{ - os << x.name(); - char delim = '['; - reflect_each(x, [&](auto&& y, auto name) { - os << delim; - os << name << "="; - stream_write_value(os, y); - delim = ','; - }); - if(delim == ',') - os << "]"; - return os; -} - -template -auto operator==(const T& x, const U& y) -> decltype(x.name() == y.name()) -{ - static_assert(is_reflectable{} or sizeof(T) <= 1, - "Missing equality operator or reflect method."); - if(x.name() != y.name()) - return false; - const auto& yy = any_cast(y); - return reflect_tie(x) == reflect_tie(yy); -} - -} // namespace operation_operators - -template -auto compute_shape_op(rank<3>, - const T& x, - const std::vector& inputs) -> decltype(x.compute_shape(inputs)) -{ - return x.compute_shape(inputs); -} - -template -auto compute_shape_op(rank<2>, const T& x, const std::vector& inputs) - -> decltype(x.normalize_compute_shape(inputs)) -{ - if(inputs.empty()) - MIGRAPHX_THROW("At least one input is required for " + x.name()); - dependent_type y = x; - normalize_attributes(y, inputs[0]); - return any_cast(y).normalize_compute_shape(inputs); -} - -template -auto compute_shape_op(rank<1>, - const T& x, - const std::vector& inputs) -> decltype(x.compute_shape(inputs, {})) -{ - return x.compute_shape(inputs, {}); -} - -template -shape compute_shape_op(rank<0>, const T& x, const std::vector&) -{ - std::string name = x.name(); - MIGRAPHX_THROW("Shape not computable: " + name); -} - -template -shape compute_shape_op(const T& x, const std::vector& inputs) -{ - return compute_shape_op(rank<3>{}, x, inputs); -} - -template -auto mod_compute_shape_op(rank<1>, - const T& x, - const std::vector& inputs, - const std::vector& mod_args) - -> decltype(x.compute_shape(inputs, mod_args)) -{ - return x.compute_shape(inputs, mod_args); -} - -template -shape mod_compute_shape_op(rank<0>, - const T& x, - const std::vector& inputs, - const std::vector& mod_args) -{ - if(mod_args.empty()) - return compute_shape_op(x, inputs); - std::string name = x.name(); - MIGRAPHX_THROW("Shape not computable: " + name); -} - -template -shape mod_compute_shape_op(const T& x, - const std::vector& inputs, - const std::vector& mod_args) -{ - return mod_compute_shape_op(rank<1>{}, x, inputs, mod_args); -} - -template -auto compute_op(rank<1>, - const T& x, - context& ctx, - const shape& output_shape, - const std::vector& input) - -> decltype(x.compute(auto_any_cast(ctx), - make_compute_output_shape(pack(x, output_shape, input)), - input)) -{ - return x.compute( - auto_any_cast(ctx), make_compute_output_shape(pack(x, output_shape, input)), input); -} - -template -argument compute_op(rank<0>, const T& x, context&, const shape&, const std::vector&) -{ - std::string name = x.name(); - MIGRAPHX_THROW("Not computable: " + name); -} - -template -argument -compute_op(const T& x, context& ctx, const shape& output_shape, const std::vector& input) -{ - return compute_op(rank<1>{}, x, ctx, output_shape, input); -} - -template -auto compute_op(rank<1>, const T& x, const shape& output_shape, const std::vector& input) - -> decltype(x.compute(make_compute_output_shape(pack(x, output_shape, input)), input)) -{ - return x.compute(make_compute_output_shape(pack(x, output_shape, input)), input); -} - -template -argument compute_op(rank<0>, const T& x, const shape&, const std::vector&) -{ - std::string name = x.name(); - MIGRAPHX_THROW("Not computable: " + name); -} - -template -argument compute_op(const T& x, const shape& output_shape, const std::vector& input) -{ - return compute_op(rank<1>{}, x, output_shape, input); -} - -template -auto compute_op(rank<1>, - const T& x, - const shape& output, - const std::vector& inputs, - const std::vector& module_args, - F f) -> decltype(x.compute(make_compute_output_shape(pack(x, output, inputs)), - inputs, - module_args, - f)) -{ - return x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs, module_args, f); -} - -template -argument compute_op(rank<0>, - const T& x, - const shape& output, - const std::vector& inputs, - const std::vector& module_args, - F) -{ - if(module_args.empty()) - return compute_op(x, output, inputs); - std::string name = x.name(); - MIGRAPHX_THROW("Not computable: " + name); -} - -template -argument compute_op(const T& x, - const shape& output, - const std::vector& inputs, - const std::vector& module_args, - F f) -{ - return compute_op(rank<1>{}, x, output, inputs, module_args, f); -} - -template -auto compute_op(rank<4>, - const T& x, - context& ctx, - const shape& output, - const std::vector& inputs, - const std::vector& module_args, - F f) -> decltype(x.compute(auto_any_cast(ctx), - make_compute_output_shape(pack(x, output, inputs)), - inputs, - module_args, - f)) -{ - return x.compute(auto_any_cast(ctx), - make_compute_output_shape(pack(x, output, inputs)), - inputs, - module_args, - f); -} - -template -auto compute_op(rank<3>, - const T& x, - context&, - const shape& output, - const std::vector& inputs, - const std::vector& module_args, - F f) -> decltype(x.compute(make_compute_output_shape(pack(x, output, inputs)), - inputs, - module_args, - f)) -{ - return x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs, module_args, f); -} - -template -auto compute_op(rank<2>, - const T& x, - context&, - const shape& output, - const std::vector& inputs, - const std::vector&, - F) -> decltype(x.compute(make_compute_output_shape(pack(x, output, inputs)), - inputs)) -{ - return x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs); -} - -template -auto compute_op(rank<1>, - const T& x, - context& ctx, - const shape& output, - const std::vector& inputs, - const std::vector&, - F) -> decltype(x.compute(auto_any_cast(ctx), - make_compute_output_shape(pack(x, output, inputs)), - inputs)) -{ - return x.compute( - auto_any_cast(ctx), make_compute_output_shape(pack(x, output, inputs)), inputs); -} - -template -argument compute_op(rank<0>, - const T& x, - context&, - const shape&, - const std::vector&, - const std::vector&, - F) -{ - std::string name = x.name(); - MIGRAPHX_THROW("Not computable: " + name); -} - -template -argument compute_op(const T& x, - context& ctx, - const shape& output, - const std::vector& inputs, - const std::vector& module_args, - F f) -{ - return compute_op(rank<4>{}, x, ctx, output, inputs, module_args, f); -} - -template -auto is_context_free_op(rank<1>, - const T& x, - const shape& output_shape, - const std::vector& input) - -> decltype(x.compute(make_compute_output_shape(pack(x, output_shape, input)), input), - std::true_type{}); - -template -auto is_context_free_op(rank<0>, const T&, const shape&, const std::vector&) - -> std::false_type; - -template -auto is_context_free_op(const T& x) - -> decltype(is_context_free_op( - rank<1>{}, x, std::declval(), std::declval>())) -{ - return {}; -} - -template -auto need_normalization_op(rank<1>, const T& x, const std::vector& inputs) - -> decltype(x.normalize_compute_shape(inputs), std::true_type{}); - -template -auto need_normalization_op(rank<0>, const T&, const std::vector&) -> std::false_type; - -template -auto need_normalization_op(const T& x) - -> decltype(need_normalization_op(rank<1>{}, x, std::declval>())) -{ - return {}; -} - -template -std::ptrdiff_t output_alias_op(const T&, const std::vector&) -{ - return -1; -} - -template -auto finalize_op( - rank<1>, T& x, context& ctx, const shape& output_shape, const std::vector& input) - -> decltype(x.finalize(auto_any_cast(ctx), output_shape, input), void()) -{ - x.finalize(auto_any_cast(ctx), output_shape, input); -} - -template -void finalize_op(rank<0>, T&, context&, const shape&, const std::vector&) -{ -} - -template -void finalize_op(T& x, context& ctx, const shape& output_shape, const std::vector& input) -{ - finalize_op(rank<1>{}, x, ctx, output_shape, input); -} - -template -auto has_finalize_op( - rank<1>, T& x, context& ctx, const shape& output_shape, const std::vector& input) - -> decltype(x.finalize(auto_any_cast(ctx), output_shape, input), std::true_type{}); - -template -auto has_finalize_op(rank<0>, T&, context&, const shape&, const std::vector&) - -> std::false_type; - -template -auto has_finalize_op(const T&) -> decltype(has_finalize_op(rank<1>{}, - std::declval(), - std::declval(), - std::declval(), - std::declval>())) -{ - return {}; -} - -template -auto compile_op( - rank<1>, T& x, context& ctx, const shape& output_shape, const std::vector& input) - -> decltype(x.compile(auto_any_cast(ctx), output_shape, input)) -{ - return x.compile(auto_any_cast(ctx), output_shape, input); -} - -template -value compile_op(rank<0>, T&, context&, const shape&, const std::vector&) -{ - return value::object{}; -} - -template -value compile_op(const T& x, - context& ctx, - const shape& output_shape, - const std::vector& input) -{ - return compile_op(rank<1>{}, x, ctx, output_shape, input); -} - -template -value attributes_op(const T&) -{ - return value::object{}; -} - -template -value to_value_op(const T& x) -{ - return migraphx::to_value(x); -} - -template -void from_value_op(T& x, const value& v) -{ - if(not(v.is_object() or (v.empty() and v.is_array()))) - MIGRAPHX_THROW("Value is not an object"); - return migraphx::from_value(v, x); -} - -template -lifetime get_lifetime_op(const T&) -{ - return lifetime::local; -} - -} // namespace detail - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT operation -{ - // - std::string name() const; - // (optional) - bool is_context_free() const; - // (optional) - bool need_normalization() const; - // (optional) - bool has_finalize() const; - // (optional) - lifetime get_lifetime() const; - // (optional) - std::ptrdiff_t output_alias(const std::vector& input) const; - // (optional) - value compile(context& ctx, const shape& output, const std::vector& input); - // (optional) - void finalize(context& ctx, const shape& output, const std::vector& input); - // (optional) - shape compute_shape(const std::vector& input) const; - // (optional) - shape compute_shape(const std::vector& inputs, - const std::vector& mod_args) const; - // (optional) - argument compute(context& ctx, const shape& output, const std::vector& input) const; - // (optional) - argument compute(const shape& output, const std::vector& input) const; - // (optional) - argument compute(const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const; - // (optional) - argument compute(context& ctx, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const; - // (optional) - value to_value() const; - // (optional) - void from_value(const value& v); - // (optional) - value attributes() const; - // - friend std::ostream& operator<<(std::ostream& os, const operation& op); - // - friend bool operator==(const operation& x, const operation& y); -}; - -#else - -struct operation -{ - private: - template - static auto private_detail_te_default_is_context_free(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.is_context_free()) - { - return private_detail_te_self.is_context_free(); - } - - template - static bool private_detail_te_default_is_context_free(float, T&& private_detail_te_self) - { - return detail::is_context_free_op(private_detail_te_self); - } - - template - static auto private_detail_te_default_need_normalization(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.need_normalization()) - { - return private_detail_te_self.need_normalization(); - } - - template - static bool private_detail_te_default_need_normalization(float, T&& private_detail_te_self) - { - return detail::need_normalization_op(private_detail_te_self); - } - - template - static auto private_detail_te_default_has_finalize(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.has_finalize()) - { - return private_detail_te_self.has_finalize(); - } - - template - static bool private_detail_te_default_has_finalize(float, T&& private_detail_te_self) - { - return detail::has_finalize_op(private_detail_te_self); - } - - template - static auto private_detail_te_default_get_lifetime(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.get_lifetime()) - { - return private_detail_te_self.get_lifetime(); - } - - template - static lifetime private_detail_te_default_get_lifetime(float, T&& private_detail_te_self) - { - return detail::get_lifetime_op(private_detail_te_self); - } - - template - static auto private_detail_te_default_output_alias(char, - T&& private_detail_te_self, - const std::vector& input) - -> decltype(private_detail_te_self.output_alias(input)) - { - return private_detail_te_self.output_alias(input); - } - - template - static std::ptrdiff_t private_detail_te_default_output_alias(float, - T&& private_detail_te_self, - const std::vector& input) - { - return detail::output_alias_op(private_detail_te_self, input); - } - - template - static auto private_detail_te_default_compile(char, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input) - -> decltype(private_detail_te_self.compile(ctx, output, input)) - { - return private_detail_te_self.compile(ctx, output, input); - } - - template - static value private_detail_te_default_compile(float, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input) - { - return detail::compile_op(private_detail_te_self, ctx, output, input); - } - - template - static auto private_detail_te_default_finalize(char, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input) - -> decltype(private_detail_te_self.finalize(ctx, output, input)) - { - private_detail_te_self.finalize(ctx, output, input); - } - - template - static void private_detail_te_default_finalize(float, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input) - { - detail::finalize_op(private_detail_te_self, ctx, output, input); - } - - template - static auto private_detail_te_default_compute_shape(char, - T&& private_detail_te_self, - const std::vector& input) - -> decltype(private_detail_te_self.compute_shape(input)) - { - return private_detail_te_self.compute_shape(input); - } - - template - static shape private_detail_te_default_compute_shape(float, - T&& private_detail_te_self, - const std::vector& input) - { - return detail::compute_shape_op(private_detail_te_self, input); - } - - template - static auto private_detail_te_default_compute_shape(char, - T&& private_detail_te_self, - const std::vector& inputs, - const std::vector& mod_args) - -> decltype(private_detail_te_self.compute_shape(inputs, mod_args)) - { - return private_detail_te_self.compute_shape(inputs, mod_args); - } - - template - static shape private_detail_te_default_compute_shape(float, - T&& private_detail_te_self, - const std::vector& inputs, - const std::vector& mod_args) - { - return detail::mod_compute_shape_op(private_detail_te_self, inputs, mod_args); - } - - template - static auto private_detail_te_default_compute(char, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input) - -> decltype(private_detail_te_self.compute(ctx, output, input)) - { - return private_detail_te_self.compute(ctx, output, input); - } - - template - static argument private_detail_te_default_compute(float, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input) - { - return detail::compute_op(private_detail_te_self, ctx, output, input); - } - - template - static auto private_detail_te_default_compute(char, - T&& private_detail_te_self, - const shape& output, - const std::vector& input) - -> decltype(private_detail_te_self.compute(output, input)) - { - return private_detail_te_self.compute(output, input); - } - - template - static argument private_detail_te_default_compute(float, - T&& private_detail_te_self, - const shape& output, - const std::vector& input) - { - return detail::compute_op(private_detail_te_self, output, input); - } - - template - static auto private_detail_te_default_compute( - char, - T&& private_detail_te_self, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function(module_ref&, - const std::unordered_map&)> run) - -> decltype(private_detail_te_self.compute(output, input, module_args, std::move(run))) - { - return private_detail_te_self.compute(output, input, module_args, std::move(run)); - } - - template - static argument private_detail_te_default_compute( - float, - T&& private_detail_te_self, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function(module_ref&, - const std::unordered_map&)> run) - { - return detail::compute_op( - private_detail_te_self, output, input, module_args, std::move(run)); - } - - template - static auto private_detail_te_default_compute( - char, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function(module_ref&, - const std::unordered_map&)> run) - -> decltype(private_detail_te_self.compute(ctx, output, input, module_args, std::move(run))) - { - return private_detail_te_self.compute(ctx, output, input, module_args, std::move(run)); - } - - template - static argument private_detail_te_default_compute( - float, - T&& private_detail_te_self, - context& ctx, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function(module_ref&, - const std::unordered_map&)> run) - { - return detail::compute_op( - private_detail_te_self, ctx, output, input, module_args, std::move(run)); - } - - template - static auto private_detail_te_default_to_value(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.to_value()) - { - return private_detail_te_self.to_value(); - } - - template - static value private_detail_te_default_to_value(float, T&& private_detail_te_self) - { - return detail::to_value_op(private_detail_te_self); - } - - template - static auto - private_detail_te_default_from_value(char, T&& private_detail_te_self, const value& v) - -> decltype(private_detail_te_self.from_value(v)) - { - private_detail_te_self.from_value(v); - } - - template - static void - private_detail_te_default_from_value(float, T&& private_detail_te_self, const value& v) - { - detail::from_value_op(private_detail_te_self, v); - } - - template - static auto private_detail_te_default_attributes(char, T&& private_detail_te_self) - -> decltype(private_detail_te_self.attributes()) - { - return private_detail_te_self.attributes(); - } - - template - static value private_detail_te_default_attributes(float, T&& private_detail_te_self) - { - return detail::attributes_op(private_detail_te_self); - } - - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().name(), - private_detail_te_default_is_context_free( - char(0), std::declval()), - private_detail_te_default_need_normalization( - char(0), std::declval()), - private_detail_te_default_has_finalize(char(0), - std::declval()), - private_detail_te_default_get_lifetime(char(0), - std::declval()), - private_detail_te_default_output_alias(char(0), - std::declval(), - std::declval&>()), - private_detail_te_default_compile(char(0), - std::declval(), - std::declval(), - std::declval(), - std::declval&>()), - private_detail_te_default_finalize(char(0), - std::declval(), - std::declval(), - std::declval(), - std::declval&>()), - private_detail_te_default_compute_shape(char(0), - std::declval(), - std::declval&>()), - private_detail_te_default_compute_shape( - char(0), - std::declval(), - std::declval&>(), - std::declval&>()), - private_detail_te_default_compute(char(0), - std::declval(), - std::declval(), - std::declval(), - std::declval&>()), - private_detail_te_default_compute(char(0), - std::declval(), - std::declval(), - std::declval&>()), - private_detail_te_default_compute( - char(0), - std::declval(), - std::declval(), - std::declval&>(), - std::declval&>(), - std::declval( - module_ref&, const std::unordered_map&)>>()), - private_detail_te_default_compute( - char(0), - std::declval(), - std::declval(), - std::declval(), - std::declval&>(), - std::declval&>(), - std::declval( - module_ref&, const std::unordered_map&)>>()), - private_detail_te_default_to_value(char(0), - std::declval()), - private_detail_te_default_from_value(char(0), - std::declval(), - std::declval()), - private_detail_te_default_attributes(char(0), - std::declval()), - static_cast(void()), - static_cast(void()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - operation() = default; - - template , - typename = typename std::enable_if< - not std::is_same, operation>{}>::type> - operation(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template , - typename = typename std::enable_if< - not std::is_same, operation>{}>::type> - operation& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - operation rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::string name() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().name(); - } - - bool is_context_free() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().is_context_free(); - } - - bool need_normalization() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().need_normalization(); - } - - bool has_finalize() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().has_finalize(); - } - - lifetime get_lifetime() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_lifetime(); - } - - std::ptrdiff_t output_alias(const std::vector& input) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().output_alias(input); - } - - value compile(context& ctx, const shape& output, const std::vector& input) - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compile(ctx, output, input); - } - - void finalize(context& ctx, const shape& output, const std::vector& input) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().finalize(ctx, output, input); - } - - shape compute_shape(const std::vector& input) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compute_shape(input); - } - - shape compute_shape(const std::vector& inputs, - const std::vector& mod_args) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compute_shape(inputs, mod_args); - } - - argument compute(context& ctx, const shape& output, const std::vector& input) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compute(ctx, output, input); - } - - argument compute(const shape& output, const std::vector& input) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compute(output, input); - } - - argument compute(const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compute( - output, input, module_args, std::move(run)); - } - - argument compute(context& ctx, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().compute( - ctx, output, input, module_args, std::move(run)); - } - - value to_value() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().to_value(); - } - - void from_value(const value& v) - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().from_value(v); - } - - value attributes() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().attributes(); - } - - friend std::ostream& operator<<(std::ostream& os, const operation& op) - { - assert(op.private_detail_te_handle_mem_var); - return op.private_detail_te_get_handle().operator_shift_left(os); - } - - friend bool operator==(const operation& x, const operation& y) - { - assert(x.private_detail_te_handle_mem_var); - return x.private_detail_te_get_handle().operator==(y); - } - - friend bool is_shared(const operation& private_detail_x, const operation& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::string name() const = 0; - virtual bool is_context_free() const = 0; - virtual bool need_normalization() const = 0; - virtual bool has_finalize() const = 0; - virtual lifetime get_lifetime() const = 0; - virtual std::ptrdiff_t output_alias(const std::vector& input) const = 0; - virtual value - compile(context& ctx, const shape& output, const std::vector& input) = 0; - virtual void - finalize(context& ctx, const shape& output, const std::vector& input) = 0; - virtual shape compute_shape(const std::vector& input) const = 0; - virtual shape compute_shape(const std::vector& inputs, - const std::vector& mod_args) const = 0; - virtual argument - compute(context& ctx, const shape& output, const std::vector& input) const = 0; - virtual argument compute(const shape& output, const std::vector& input) const = 0; - virtual argument - compute(const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const = 0; - virtual argument - compute(context& ctx, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const = 0; - virtual value to_value() const = 0; - virtual void from_value(const value& v) = 0; - virtual value attributes() const = 0; - virtual std::ostream& operator_shift_left(std::ostream& os) const = 0; - virtual bool operator==(const operation& y) const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::string name() const override { return private_detail_te_value.name(); } - - bool is_context_free() const override - { - - return private_detail_te_default_is_context_free(char(0), private_detail_te_value); - } - - bool need_normalization() const override - { - - return private_detail_te_default_need_normalization(char(0), private_detail_te_value); - } - - bool has_finalize() const override - { - - return private_detail_te_default_has_finalize(char(0), private_detail_te_value); - } - - lifetime get_lifetime() const override - { - - return private_detail_te_default_get_lifetime(char(0), private_detail_te_value); - } - - std::ptrdiff_t output_alias(const std::vector& input) const override - { - - return private_detail_te_default_output_alias(char(0), private_detail_te_value, input); - } - - value compile(context& ctx, const shape& output, const std::vector& input) override - { - - return private_detail_te_default_compile( - char(0), private_detail_te_value, ctx, output, input); - } - - void finalize(context& ctx, const shape& output, const std::vector& input) override - { - - private_detail_te_default_finalize( - char(0), private_detail_te_value, ctx, output, input); - } - - shape compute_shape(const std::vector& input) const override - { - - return private_detail_te_default_compute_shape(char(0), private_detail_te_value, input); - } - - shape compute_shape(const std::vector& inputs, - const std::vector& mod_args) const override - { - - return private_detail_te_default_compute_shape( - char(0), private_detail_te_value, inputs, mod_args); - } - - argument compute(context& ctx, - const shape& output, - const std::vector& input) const override - { - - return private_detail_te_default_compute( - char(0), private_detail_te_value, ctx, output, input); - } - - argument compute(const shape& output, const std::vector& input) const override - { - - return private_detail_te_default_compute( - char(0), private_detail_te_value, output, input); - } - - argument compute( - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const override - { - - return private_detail_te_default_compute( - char(0), private_detail_te_value, output, input, module_args, std::move(run)); - } - - argument compute( - context& ctx, - const shape& output, - const std::vector& input, - const std::vector& module_args, - std::function( - module_ref&, const std::unordered_map&)> run) const override - { - - return private_detail_te_default_compute( - char(0), private_detail_te_value, ctx, output, input, module_args, std::move(run)); - } - - value to_value() const override - { - - return private_detail_te_default_to_value(char(0), private_detail_te_value); - } - - void from_value(const value& v) override - { - - private_detail_te_default_from_value(char(0), private_detail_te_value, v); - } - - value attributes() const override - { - - return private_detail_te_default_attributes(char(0), private_detail_te_value); - } - - std::ostream& operator_shift_left(std::ostream& os) const override - { - using migraphx::detail::operation_operators::operator<<; - return os << private_detail_te_value; - } - - bool operator==(const operation& y) const override - { - using migraphx::detail::operation_operators::operator==; - return private_detail_te_value == y; - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const operation* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(operation* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(operation& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const operation& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -inline bool operator!=(const operation& x, const operation& y) { return not(x == y); } - -inline value -compile(operation& op, context& ctx, const shape& output_shape, const std::vector& input) -{ - return op.compile(ctx, output_shape, input); -} -template -inline value -compile(operation& op, Context& ctx, const shape& output_shape, const std::vector& input) -{ - dependent_type ctx2 = std::ref(ctx); - return compile(op, ctx2, output_shape, input); -} -template -inline auto compile(T& op, Context& ctx, const shape& output_shape, const std::vector& input) - -> decltype(op.compile(ctx, ctx, output_shape, input)) -{ - return op.compile(ctx, ctx, output_shape, input); -} -inline shape compute_shape(const operation& op, const std::vector& inputs) -{ - return op.compute_shape(inputs); -} - -template -inline auto compute_shape(const T& op, - const std::vector& inputs) -> decltype(op.compute_shape(inputs)) -{ - return op.compute_shape(inputs); -} - -template -inline auto compute_shape(const T& op, const std::vector& inputs) - -> decltype(op.normalize_compute_shape(inputs)) -{ - return detail::compute_shape_op(op, inputs); -} - -inline shape compute_shape(const operation& op, - const std::vector& inputs, - const std::vector& mod_args) -{ - return op.compute_shape(inputs, mod_args); -} - -template -inline auto compute_shape(const T& op, - const std::vector& inputs, - const std::vector& mod_args) - -> decltype(op.compute_shape(inputs, mod_args)) -{ - return op.compute_shape(inputs, mod_args); -} - -template -inline auto compute_shape(const T& op, - const std::vector& inputs, - const std::vector& mod_args) - -> decltype(op.normalize_compute_shape(inputs, mod_args)) -{ - return detail::compute_shape_op(op, inputs, mod_args); -} - -inline bool is_context_free(const operation& op) { return op.is_context_free(); } - -template -bool is_context_free(const T& x) -{ - return detail::is_context_free_op(x); -} - -inline bool need_normalization(const operation& op) { return op.need_normalization(); } - -template -bool need_normalization(const T& x) -{ - return detail::need_normalization_op(x); -} - -inline bool has_finalize(const operation& op) { return op.has_finalize(); } - -template -bool has_finalize(const T& x) -{ - return detail::has_finalize_op(x); -} - -MIGRAPHX_EXPORT void migraphx_to_value(value& v, const operation& op); -MIGRAPHX_EXPORT void migraphx_from_value(const value& v, operation& op); - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/operators.hpp b/docker/rocm/migraphx/include/migraphx/operators.hpp deleted file mode 100644 index b183e7957..000000000 --- a/docker/rocm/migraphx/include/migraphx/operators.hpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_HPP -#define MIGRAPHX_GUARD_OPERATORS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/optimize_module.hpp b/docker/rocm/migraphx/include/migraphx/optimize_module.hpp deleted file mode 100644 index 9c309025b..000000000 --- a/docker/rocm/migraphx/include/migraphx/optimize_module.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_OPTIMIZE_MODULE_HPP -#define MIGRAPHX_GUARD_RTGLIB_OPTIMIZE_MODULE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -/** - * Runs several passes in a loop - */ -struct MIGRAPHX_EXPORT optimize_module -{ - std::unordered_set propagate_constant_skip_ops = {}; - std::string name() const { return "optimize_module"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/optional.hpp b/docker/rocm/migraphx/include/migraphx/optional.hpp deleted file mode 100644 index 5f3be003e..000000000 --- a/docker/rocm/migraphx/include/migraphx/optional.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_OPTIONAL_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_OPTIONAL_HPP - -#include - -#if defined(CPPCHECK) -#define MIGRAPHX_HAS_OPTIONAL 1 -#define MIGRAPHX_HAS_OPTIONAL_TS 1 -#elif defined(_WIN32) -#if _MSC_VER >= 1920 -#define MIGRAPHX_HAS_OPTIONAL 1 -#define MIGRAPHX_HAS_OPTIONAL_TS 0 -#elif _MSC_VER >= 1900 -#define MIGRAPHX_HAS_OPTIONAL 0 -#define MIGRAPHX_HAS_OPTIONAL_TS 1 -#else -#define MIGRAPHX_HAS_OPTIONAL 0 -#define MIGRAPHX_HAS_OPTIONAL_TS 0 -#endif -#elif defined(__has_include) -#if __has_include() && __cplusplus >= 201703L -#define MIGRAPHX_HAS_OPTIONAL 1 -#else -#define MIGRAPHX_HAS_OPTIONAL 0 -#endif -#if __has_include() && __cplusplus >= 201103L -#define MIGRAPHX_HAS_OPTIONAL_TS 1 -#else -#define MIGRAPHX_HAS_OPTIONAL_TS 0 -#endif -#else -#define MIGRAPHX_HAS_OPTIONAL 0 -#define MIGRAPHX_HAS_OPTIONAL_TS 0 -#endif - -#if MIGRAPHX_HAS_OPTIONAL -#include -#elif MIGRAPHX_HAS_OPTIONAL_TS -#include -#else -#error "No optional include available" -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#if MIGRAPHX_HAS_OPTIONAL -template -using optional = std::optional; -using nullopt_t = std::nullopt_t; -constexpr auto nullopt = std::nullopt; -#elif MIGRAPHX_HAS_OPTIONAL_TS -template -using optional = std::experimental::optional; -using nullopt_t = std::experimental::nullopt_t; -constexpr auto nullopt = std::experimental::nullopt; -#endif - -template -bool has_value(const optional& x) -{ - return x != nullopt; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_MIGRAPHX_OPTIONAL_HPP diff --git a/docker/rocm/migraphx/include/migraphx/output_iterator.hpp b/docker/rocm/migraphx/include/migraphx/output_iterator.hpp deleted file mode 100644 index e4d670b85..000000000 --- a/docker/rocm/migraphx/include/migraphx/output_iterator.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_OUTPUT_ITERATOR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_OUTPUT_ITERATOR_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct function_output_iterator -{ - copy_assignable_function f; - - using self = function_output_iterator; - using difference_type = void; - using reference = void; - using value_type = void; - using pointer = void; - using iterator_category = std::output_iterator_tag; - - struct output_proxy - { - template - output_proxy& operator=(const T& value) - { - assert(f); - (*f)(value); - return *this; - } - copy_assignable_function* f; - }; - output_proxy operator*() { return output_proxy{&f}; } - self& operator++() { return *this; } - self& operator++(int) { return *this; } // NOLINT -}; - -template -function_output_iterator make_function_output_iterator(F f) -{ - return {std::move(f)}; -} - -template -auto join_back_inserter(Container& c) -{ - return make_function_output_iterator( - [&](const auto& r) { c.insert(c.end(), r.begin(), r.end()); }); -} - -template -auto push_inserter(Container& c) -{ - return make_function_output_iterator([&](const auto& x) { c.push(x); }); -} -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_MIGRAPHX_OUTPUT_ITERATOR_HPP diff --git a/docker/rocm/migraphx/include/migraphx/pad_calc.hpp b/docker/rocm/migraphx/include/migraphx/pad_calc.hpp deleted file mode 100644 index cb5972fb6..000000000 --- a/docker/rocm/migraphx/include/migraphx/pad_calc.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_PAD_CALC_HPP -#define MIGRAPHX_GUARD_OPERATORS_PAD_CALC_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT -void calculate_padding(int64_t idx, - std::vector& pads, - int64_t input_dim, - int64_t stride, - int64_t dilation, - int64_t weight_dim, - bool is_same_upper = true); - -/*! - * Calculate the padding for auto_padding. Used for dynamic shapes - * where the padding calculation must be done at evaluation time. - * \return padding in the form of {x0_begin, x1_begin, ... x0_end , x1_end, ...} - */ -MIGRAPHX_EXPORT -std::vector calc_dyn_auto_pad(const std::vector& input_lens, - const std::vector& wei_lens, - const std::vector& strides, - const std::vector& dilations, - bool use_upper); - -// Used for dynamic auto padding of convolution operators since padding needs to be computed at -// evaulation time. -MIGRAPHX_EXPORT -shape compute_padded_shape(const shape& input, - const shape& weights, - const std::vector& padding, - const std::vector& stride, - const std::vector& dilation); - -// Used for dynamic auto padding of pooling operators where padding needs to be computed at -// evaulation time. -MIGRAPHX_EXPORT -shape compute_padded_pool_shape(const shape& input, - const shape& kernel, - const std::vector& padding, - const std::vector& stride, - const std::vector& dilation); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/par.hpp b/docker/rocm/migraphx/include/migraphx/par.hpp deleted file mode 100644 index b448f39fc..000000000 --- a/docker/rocm/migraphx/include/migraphx/par.hpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_PAR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_PAR_HPP - -#include -#if MIGRAPHX_HAS_EXECUTORS -#include -#else -#include -#endif -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace detail { - -struct exception_list -{ - std::vector exceptions; - std::mutex m; - void add_exception() - { - std::lock_guard guard(m); - exceptions.push_back(std::current_exception()); - } - template - auto collect(F f) - { - return [f, this](auto&&... xs) { - try - { - f(std::forward(xs)...); - } - catch(...) - { - this->add_exception(); - } - }; - } - void throw_if_exception() const - { - if(not exceptions.empty()) - std::rethrow_exception(exceptions.front()); - } -}; - -} // namespace detail - -template -OutputIt par_transform(InputIt first1, InputIt last1, OutputIt d_first, UnaryOperation unary_op) -{ -#if MIGRAPHX_HAS_EXECUTORS - return std::transform(std::execution::par, first1, last1, d_first, std::move(unary_op)); -#else - return std::transform(first1, last1, d_first, std::move(unary_op)); -#endif -} - -template -OutputIt par_transform( - InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOperation binary_op) -{ -#if MIGRAPHX_HAS_EXECUTORS - return std::transform( - std::execution::par, first1, last1, first2, d_first, std::move(binary_op)); -#else - return std::transform(first1, last1, first2, d_first, std::move(binary_op)); -#endif -} - -template -void par_for_each(InputIt first, InputIt last, UnaryFunction f) -{ -#if MIGRAPHX_HAS_EXECUTORS - // Propagate the exception - detail::exception_list ex; - std::for_each(std::execution::par, first, last, ex.collect(std::move(f))); - ex.throw_if_exception(); -#else - simple_par_for(last - first, [&](auto i) { f(first[i]); }); -#endif -} - -template -auto par_copy_if(Ts&&... xs) -{ -#if MIGRAPHX_HAS_EXECUTORS - return std::copy_if(std::execution::par, std::forward(xs)...); -#else - return std::copy_if(std::forward(xs)...); -#endif -} - -template -auto par_sort(Ts&&... xs) -{ -#if MIGRAPHX_HAS_EXECUTORS - return std::sort(std::execution::par, std::forward(xs)...); -#else - return std::sort(std::forward(xs)...); -#endif -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_PAR_HPP diff --git a/docker/rocm/migraphx/include/migraphx/par_dfor.hpp b/docker/rocm/migraphx/include/migraphx/par_dfor.hpp deleted file mode 100644 index 8a9f20de6..000000000 --- a/docker/rocm/migraphx/include/migraphx/par_dfor.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PAR_DFOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_PAR_DFOR_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -auto par_dfor(Ts... xs) -{ - return [=](auto f) { - using array_type = std::array; - array_type lens = {{static_cast(xs)...}}; - auto n = std::accumulate(lens.begin(), lens.end(), 1, std::multiplies{}); - const std::size_t min_grain = 8; - if(n > 2 * min_grain) - { - array_type strides; - strides.fill(1); - std::partial_sum(lens.rbegin(), - lens.rend() - 1, - strides.rbegin() + 1, - std::multiplies()); - auto size = - std::accumulate(lens.begin(), lens.end(), 1, std::multiplies()); - par_for(size, min_grain, [&](std::size_t i) { - array_type indices; - std::transform(strides.begin(), - strides.end(), - lens.begin(), - indices.begin(), - [&](size_t stride, size_t len) { return (i / stride) % len; }); - migraphx::unpack(f, indices); - }); - } - else - { - dfor(xs...)(f); - } - }; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/par_for.hpp b/docker/rocm/migraphx/include/migraphx/par_for.hpp deleted file mode 100644 index d34b6c5c1..000000000 --- a/docker/rocm/migraphx/include/migraphx/par_for.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PAR_FOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_PAR_FOR_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -void par_for(std::size_t n, F f) -{ - using iterator = basic_iota_iterator; - par_for_each(iterator{0, {}}, iterator{n, {}}, f); -} - -template -void par_for(std::size_t n, std::size_t min_grain, F f) -{ - simple_par_for(n, min_grain, f); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/param_utils.hpp b/docker/rocm/migraphx/include/migraphx/param_utils.hpp deleted file mode 100644 index 6ce2fce38..000000000 --- a/docker/rocm/migraphx/include/migraphx/param_utils.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_PARAM_UTILS_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_PARAM_UTILS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT std::string param_name(std::size_t i, const std::string& prefix = "x"); - -void sort_params(std::vector& params); - -// Find the inputs for a module by finding instructions that are mapped to the -// parameters in the module -std::vector - MIGRAPHX_EXPORT find_inputs(const std::unordered_map& map_ins, - const_module_ref parent, - const_module_ref sub); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_PARAM_UTILS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/pass.hpp b/docker/rocm/migraphx/include/migraphx/pass.hpp deleted file mode 100644 index 696cc199a..000000000 --- a/docker/rocm/migraphx/include/migraphx/pass.hpp +++ /dev/null @@ -1,379 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_PASS_HPP -#define MIGRAPHX_GUARD_PASS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct program; -struct module; -struct module_pass_manager; - -#ifdef DOXYGEN - -/// An interface for applying a transformation to the instructions in a -/// `program` -struct pass -{ - /// A unique name used to identify the pass - std::string name() const; - /// Run the pass on the module - void apply(module_pass_manager& mpm) const; - void apply(module& m) const; - /// Run the pass on the program - void apply(program& p) const; -}; - -#else - -MIGRAPHX_EXPORT module& get_module(module_pass_manager& mpm); - -namespace detail { - -template -auto module_pass_manager_apply(rank<1>, - const T& x, - module_pass_manager& mpm) -> decltype(x.apply(get_module(mpm))) -{ - return x.apply(get_module(mpm)); -} - -template -void module_pass_manager_apply(rank<0>, const T&, module_pass_manager&) -{ -} - -template -void module_pass_manager_apply(const T& x, module_pass_manager& mpm) -{ - module_pass_manager_apply(rank<1>{}, x, mpm); -} - -} // namespace detail - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT pass -{ - // - std::string name() const; - // (optional) - void apply(module_pass_manager& mpm) const; - // (optional) - void apply(program& p) const; -}; - -#else - -struct pass -{ - private: - template - static auto - private_detail_te_default_apply(char, T&& private_detail_te_self, module_pass_manager& mpm) - -> decltype(private_detail_te_self.apply(mpm)) - { - private_detail_te_self.apply(mpm); - } - - template - static void - private_detail_te_default_apply(float, T&& private_detail_te_self, module_pass_manager& mpm) - { - migraphx::detail::module_pass_manager_apply(private_detail_te_self, mpm); - } - - template - static auto private_detail_te_default_apply(char, T&& private_detail_te_self, program& p) - -> decltype(private_detail_te_self.apply(p)) - { - private_detail_te_self.apply(p); - } - - template - static void private_detail_te_default_apply(float, T&& private_detail_te_self, program& p) - { - migraphx::nop(private_detail_te_self, p); - } - - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().name(), - private_detail_te_default_apply(char(0), - std::declval(), - std::declval()), - private_detail_te_default_apply( - char(0), std::declval(), std::declval()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - pass() = default; - - template , - typename = typename std::enable_if< - not std::is_same, pass>{}>::type> - pass(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template , - typename = typename std::enable_if< - not std::is_same, pass>{}>::type> - pass& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - pass rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::string name() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().name(); - } - - void apply(module_pass_manager& mpm) const - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().apply(mpm); - } - - void apply(program& p) const - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().apply(p); - } - - friend bool is_shared(const pass& private_detail_x, const pass& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::string name() const = 0; - virtual void apply(module_pass_manager& mpm) const = 0; - virtual void apply(program& p) const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::string name() const override { return private_detail_te_value.name(); } - - void apply(module_pass_manager& mpm) const override - { - - private_detail_te_default_apply(char(0), private_detail_te_value, mpm); - } - - void apply(program& p) const override - { - - private_detail_te_default_apply(char(0), private_detail_te_value, p); - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const pass* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(pass* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(pass& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const pass& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -/// Used in the targets to enable/disable compiler passes -MIGRAPHX_EXPORT pass enable_pass(bool enabled, pass p); - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/pass_manager.hpp b/docker/rocm/migraphx/include/migraphx/pass_manager.hpp deleted file mode 100644 index 535c7b5a3..000000000 --- a/docker/rocm/migraphx/include/migraphx/pass_manager.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_PASS_MANAGER_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_PASS_MANAGER_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager -{ - module_pass_manager() = default; - module_pass_manager(const module_pass_manager&) = delete; - virtual module& get_module() = 0; - virtual module* create_module(const std::string& name) = 0; - virtual module* create_module(const std::string& name, module m) = 0; - virtual void rename_module(const std::string& old_name, const std::string& new_name) = 0; - virtual module* get_common_parent() = 0; - virtual module* get_root_module() = 0; - virtual void run_pass(const pass& p) = 0; - - protected: - virtual ~module_pass_manager() {} -}; - -MIGRAPHX_EXPORT void run_passes(program& prog, - module_ref root_mod, - const std::vector& passes, - tracer trace = tracer{}); -MIGRAPHX_EXPORT void -run_passes(module& mod, const std::vector& passes, tracer trace = tracer{}); -MIGRAPHX_EXPORT void -run_passes(program& prog, const std::vector& passes, tracer trace = tracer{}); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/permutation.hpp b/docker/rocm/migraphx/include/migraphx/permutation.hpp deleted file mode 100644 index d368e1d42..000000000 --- a/docker/rocm/migraphx/include/migraphx/permutation.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PERMUTATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_PERMUTATION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -inline Vector reorder_dims(const Vector& dims, const std::vector& permutation) -{ - Vector result(dims.size()); - assert(dims.size() == permutation.size()); - for(std::size_t i = 0; i < dims.size(); i++) - { - result[i] = dims[permutation[i]]; - } - return result; -} - -MIGRAPHX_EXPORT shape reorder_shape(const shape& s, const std::vector& permutation); - -template -inline std::vector sort_permutation(const Vector& data, Op op) -{ - std::vector result(data.size()); - std::iota(result.begin(), result.end(), 0); - std::stable_sort( - result.begin(), result.end(), [&](auto x, auto y) { return op(data[x], data[y]); }); - return result; -} - -/*! - * Returns the inverse permutation that could be applied to undo the inputted permutation - */ -MIGRAPHX_EXPORT std::vector invert_permutation(const std::vector& permutation); - -/*! - * Finds the permutation that would make the shape not transposed (refering to shape.transposed()) - */ -MIGRAPHX_EXPORT std::vector find_permutation(const shape& s); -MIGRAPHX_EXPORT std::vector find_permutation(const std::vector& shapes); - -/// Normalize the shapes so the order of dimensions will be in the order it is -/// in memory as much as possible. -MIGRAPHX_EXPORT std::vector normalize_permutation(const std::vector& shapes); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/preallocate_param.hpp b/docker/rocm/migraphx/include/migraphx/preallocate_param.hpp deleted file mode 100644 index b66738142..000000000 --- a/docker/rocm/migraphx/include/migraphx/preallocate_param.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_PREALLOCATE_PARAM_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_PREALLOCATE_PARAM_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct MIGRAPHX_EXPORT preallocate_param -{ - std::string param; - allocation_model model; - std::string name() const { return "preallocate_param"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_PREALLOCATE_PARAM_HPP diff --git a/docker/rocm/migraphx/include/migraphx/process.hpp b/docker/rocm/migraphx/include/migraphx/process.hpp deleted file mode 100644 index b0a25741b..000000000 --- a/docker/rocm/migraphx/include/migraphx/process.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_PROCESS_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_PROCESS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct process_impl; - -struct MIGRAPHX_EXPORT process -{ - using writer = std::function; - explicit process(const std::string& cmd, const std::vector& args = {}); - - explicit process(const fs::path& cmd, const std::vector& args = {}) - : process{cmd.string(), args} - { - } - - // move constructor - process(process&&) noexcept; - - // copy assignment operator - process& operator=(process rhs); - - ~process() noexcept; - - process& cwd(const fs::path& p); - process& env(const std::vector& envs); - - void exec(); - void write(std::function pipe_in); - void read(const writer& output) const; - - private: - std::unique_ptr impl; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_PROCESS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/program.hpp b/docker/rocm/migraphx/include/migraphx/program.hpp deleted file mode 100644 index 2a82f381c..000000000 --- a/docker/rocm/migraphx/include/migraphx/program.hpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_PROGRAM_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_PROGRAM_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_COMPILE) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_EVAL) - -struct program_impl; - -struct marker; - -/** - * @brief Stores the instruction stream - */ -struct MIGRAPHX_EXPORT program -{ - program(); - - explicit program(module m); - - // move constructor - program(program&&) noexcept; - - // copy constructor - program(const program&); - - // copy assignment operator - program& operator=(program); - - ~program() noexcept; - - std::vector get_parameter_names() const; - - shape get_parameter_shape(std::string name) const; - - instruction_ref get_parameter(std::string name) const; - - std::unordered_map get_parameter_shapes() const; - - std::vector eval(parameter_map params, - execution_environment exec_env = execution_environment{}) const; - - std::vector eval_with_context(std::vector& ctx, parameter_map params) const; - - void finish() const; - - std::size_t size() const; - - std::vector get_output_shapes() const; - - context& get_context() const; - - instruction_ref validate() const; - - target_assignments get_target_assignments(const std::vector& targets, - assignment_options options = assignment_options{}); - - void compile(const target& t, compile_options options = compile_options{}); - - void compile(const std::vector& targets, - std::vector compile_opts = {}); - - bool is_compiled() const; - - void finalize(); - - void perf_report(std::ostream& os, - std::size_t n, - parameter_map params, - std::size_t batch = 1, - bool detailed = false) const; - - void mark(const parameter_map& params, marker m); - - value to_value() const; - void from_value(const value& v); - - void debug_print() const; - void debug_print(instruction_ref ins) const; - void print(std::unordered_map& names, - const std::function)>& - print_func) const; - void print(const std::function)>& - print_func) const; - - void print_graph(std::ostream& os, bool brief = false) const; - void print_py(std::ostream& os) const; - void print_cpp(std::ostream& os) const; - - void dry_run(parameter_map params) const; - - void annotate(std::ostream& os, const std::function& a) const; - - program& sort(); - - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const program& p); - MIGRAPHX_EXPORT friend bool operator==(const program& x, const program& y); - friend bool operator!=(const program& x, const program& y) { return not(x == y); } - - // module related api - module* create_module(const std::string& name); - module* create_module(const std::string& name, module m); - module* get_module(const std::string& name); - const module* get_module(const std::string& name) const; - - module* get_main_module(); - const module* get_main_module() const; - - std::vector get_modules() const; - std::vector get_modules(); - - std::unordered_multimap get_module_tree(); - - void remove_module(const std::string& name); - void rename_module(const std::string& old_name, const std::string& new_name); - void remove_unused_modules(); - - private: - void assign(const program& p); - std::unique_ptr impl; -}; -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/promote_literals.hpp b/docker/rocm/migraphx/include/migraphx/promote_literals.hpp deleted file mode 100644 index 5e32a7ec3..000000000 --- a/docker/rocm/migraphx/include/migraphx/promote_literals.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PROMOTE_LITERALS_HPP -#define MIGRAPHX_GUARD_RTGLIB_PROMOTE_LITERALS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Replace literals in submodules with literals in the root module. - * Intended to allow for reuse of the literals between submodules. - */ -struct MIGRAPHX_EXPORT promote_literals -{ - std::string name() const { return "promote_literals"; } - void apply(module_pass_manager&) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/propagate_constant.hpp b/docker/rocm/migraphx/include/migraphx/propagate_constant.hpp deleted file mode 100644 index 0b8ac56ab..000000000 --- a/docker/rocm/migraphx/include/migraphx/propagate_constant.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PROPAGATE_CONSTANT_HPP -#define MIGRAPHX_GUARD_RTGLIB_PROPAGATE_CONSTANT_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Replace instructions which take all literals with a literal of the computation. - */ -struct MIGRAPHX_EXPORT propagate_constant -{ - std::unordered_set skip_ops = {}; - std::string name() const { return "propagate_constant"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/quantization.hpp b/docker/rocm/migraphx/include/migraphx/quantization.hpp deleted file mode 100644 index eead5e40b..000000000 --- a/docker/rocm/migraphx/include/migraphx/quantization.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_QUANTIZATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_QUANTIZATION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct program; - -MIGRAPHX_EXPORT void quantize_fp16(program& prog, - const std::vector& ins_names = {"all"}); - -MIGRAPHX_EXPORT void quantize_int8(program& prog, - const target& t, - const std::vector& calibration, - const std::unordered_set& ins_names = { - "dot", "convolution"}); -MIGRAPHX_EXPORT void -quantize_fp8(program& prog, const target& t, const std::vector& calibration); - -MIGRAPHX_EXPORT void quantize_int4_weights(program& prog); - -MIGRAPHX_EXPORT void quantize_bf16(program& prog, - const std::vector& ins_names = {"all"}); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/quantize_8bits.hpp b/docker/rocm/migraphx/include/migraphx/quantize_8bits.hpp deleted file mode 100644 index 934502583..000000000 --- a/docker/rocm/migraphx/include/migraphx/quantize_8bits.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_QUANTIZE_8BITS_HPP -#define MIGRAPHX_GUARD_RTGLIB_QUANTIZE_8BITS_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct program; -struct module; - -/** - * capture inputs of operators to be quantized to int8 or fp8 - */ -struct MIGRAPHX_EXPORT capture_arguments_pass -{ - std::unordered_set ins_names = {"dot", "convolution"}; - std::function)> f{}; - std::size_t* param_index = nullptr; - std::string name() const { return "capture_arguments"; } - void apply(module& m) const; -}; - -/** - * quantize a program to int8 or fp8 - */ -struct MIGRAPHX_EXPORT quantize_8bits_pass -{ - shape::type_t precision = shape::int8_type; - std::vector> quant_params; - std::string name() const { return "quantize_8bits"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/quantize_int4.hpp b/docker/rocm/migraphx/include/migraphx/quantize_int4.hpp deleted file mode 100644 index 586ad3f72..000000000 --- a/docker/rocm/migraphx/include/migraphx/quantize_int4.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_QUANTIZE_INT4_HPP -#define MIGRAPHX_GUARD_RTGLIB_QUANTIZE_INT4_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct program; -struct module; - -/** - * quantize a program to int4 - */ -struct MIGRAPHX_EXPORT quantize_int4_pass -{ - std::vector ins_names; - std::string name() const { return "quantize_int4"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/ranges.hpp b/docker/rocm/migraphx/include/migraphx/ranges.hpp deleted file mode 100644 index 7ce6d809b..000000000 --- a/docker/rocm/migraphx/include/migraphx/ranges.hpp +++ /dev/null @@ -1,286 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_RANGES_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_RANGES_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace detail { - -template -auto generic_find_impl(rank<2>, String&& s, const T& x) -> decltype(s.npos, s.begin() + s.find(x)) -{ - auto index = s.find(x); - if(index == s.npos) - return s.end(); - else - return s.begin() + index; -} - -template -auto generic_find_impl(rank<1>, C&& c, const T& x) -> decltype(c.find(x)) -{ - return c.find(x); -} - -template -auto generic_find_impl(rank<0>, C&& c, const T& x) -{ - return std::find(c.begin(), c.end(), x); -} - -template -auto generic_find_at_impl(rank<1>, C&& c, const T& x) -> decltype(c.find(x)) -{ - return c.find(x); -} - -template -auto generic_find_at_impl(rank<0>, C&& c, const T& x) -{ - auto n = std::distance(c.begin(), c.end()); - if(x >= n) - return c.end(); - return std::next(c.begin(), x); -} - -template -decltype(auto) generic_at_impl(rank<1>, const C&, T&& it) -{ - return it->second; -} - -template -decltype(auto) generic_at_impl(rank<0>, const C&, T&& it) -{ - return *it; -} - -struct empty -{ -}; - -} // namespace detail - -template -auto generic_find(C&& c, const T& x) -{ - return detail::generic_find_impl(rank<2>{}, c, x); -} - -template -decltype(auto) at(C&& c, const T& x, const std::string& msg = "") -{ - auto it = detail::generic_find_at_impl(rank<2>{}, c, x); - if(it == c.end()) - { - if(msg.empty()) - MIGRAPHX_THROW("At operator out of range for " + get_type_name(c)); - else - MIGRAPHX_THROW(msg); - } - return detail::generic_at_impl(rank<2>{}, c, it); -} - -template -bool contains(const C& c, const T& x) -{ - return generic_find(c, x) != c.end(); -} - -template -bool contains(const std::initializer_list& c, const T& x) -{ - return generic_find(c, x) != c.end(); -} - -template -bool contains(const std::initializer_list& c, const U& x) -{ - return generic_find(c, x) != c.end(); -} - -template -bool all_of(const C& c, const Predicate& p) -{ - return std::all_of(c.begin(), c.end(), p); -} - -template -bool all_of(const std::initializer_list& c, const Predicate& p) -{ - return std::all_of(c.begin(), c.end(), p); -} - -template -bool all_of(detail::empty, const Predicate&) -{ - return true; -} - -template -bool any_of(const C& c, const Predicate& p) -{ - return std::any_of(c.begin(), c.end(), p); -} - -template -bool any_of(const std::initializer_list& c, const Predicate& p) -{ - return std::any_of(c.begin(), c.end(), p); -} - -template -bool any_of(detail::empty, const Predicate&) -{ - return false; -} - -template -bool none_of(const C& c, const Predicate& p) -{ - return std::none_of(c.begin(), c.end(), p); -} - -template -bool none_of(const std::initializer_list& c, const Predicate& p) -{ - return std::none_of(c.begin(), c.end(), p); -} - -template -bool none_of(detail::empty, const Predicate&) -{ - return true; -} - -template -auto find_if(const C& c, const Predicate& p) -{ - return std::find_if(c.begin(), c.end(), p); -} - -template -void copy(Range&& r, Iterator it) -{ - std::copy(r.begin(), r.end(), it); -} - -template -void copy_if(Range&& r, Iterator it, Predicate pred) -{ - std::copy_if(r.begin(), r.end(), it, pred); -} - -template -void transform(Range&& r, Iterator it, F f) -{ - std::transform(r.begin(), r.end(), it, f); -} - -template -void transform(Range1&& r1, Range2&& r2, Iterator it, F f) -{ - std::transform(r1.begin(), r1.end(), r2.begin(), it, f); -} - -template -auto reverse(Range&& r) -{ - return range(std::make_reverse_iterator(r.end()), std::make_reverse_iterator(r.begin())); -} - -template -void replace(Range&& r, const T& old, const T& new_x) -{ - std::replace(r.begin(), r.end(), old, new_x); -} - -template -bool equal(R1&& r1, R2&& r2, Predicate... pred) -{ - return std::equal(r1.begin(), r1.end(), r2.begin(), r2.end(), pred...); -} - -template -auto distance(Range&& r) -{ - return std::distance(r.begin(), r.end()); -} - -template -using range_value = std::decay_t().begin())>; - -template -std::vector> find_all(Range&& r, Predicate p) -{ - std::vector> result; - std::copy_if(r.begin(), r.end(), std::back_inserter(result), p); - return result; -} - -template -struct iterator_range -{ - Iterator start; - Iterator last; - - Iterator begin() const { return start; } - - Iterator end() const { return last; } -}; - -template {})> -iterator_range range(Iterator start, Iterator last) -{ - return {start, last}; -} - -inline iterator_range range(std::ptrdiff_t start, std::ptrdiff_t last) -{ - return {{start, {}}, {last, {}}}; -} -inline iterator_range range(std::ptrdiff_t last) { return range(0, last); } - -template -iterator_range range(std::pair p) -{ - return {p.first, p.second}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rank.hpp b/docker/rocm/migraphx/include/migraphx/rank.hpp deleted file mode 100644 index 826f93e41..000000000 --- a/docker/rocm/migraphx/include/migraphx/rank.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_RANK_HPP -#define MIGRAPHX_GUARD_RTGLIB_RANK_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct rank : rank -{ -}; - -template <> -struct rank<0> -{ -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/raw_data.hpp b/docker/rocm/migraphx/include/migraphx/raw_data.hpp deleted file mode 100644 index 3d3a7be08..000000000 --- a/docker/rocm/migraphx/include/migraphx/raw_data.hpp +++ /dev/null @@ -1,307 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RAW_DATA_HPP -#define MIGRAPHX_GUARD_RAW_DATA_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct raw_data_base -{ -}; - -/** - * @brief Provides a base class for common operations with raw buffer - * - * For classes that handle a raw buffer of data, this will provide common operations such as equals, - * printing, and visitors. To use this class the derived class needs to provide a `data()` method to - * retrieve a raw pointer to the data, and `get_shape` method that provides the shape of the data. - * - */ -template -struct raw_data : raw_data_base -{ - template - friend Stream& operator<<(Stream& os, const Derived& d) - { - if(not d.empty()) - d.visit([&](auto x) { os << x; }, - [&](auto&& xs) { - for(auto&& x : xs) - { - os << "{ "; - os << x; - os << " }, "; - } - }); - return os; - } - - /** - * @brief Visits a single data element at a certain index. - * - * @param v A function which will be called with the type of data - * @param n The index to read from - */ - template - void visit_at(Visitor v, std::size_t n = 0) const - { - auto&& derived = static_cast(*this); - if(derived.empty()) - MIGRAPHX_THROW("Visiting empty data!"); - auto&& s = derived.get_shape(); - s.visit_type([&](auto as) { v(*(as.from(derived.data()) + s.index(n))); }); - } - - template - void visit(Visitor v, TupleVisitor tv) const - { - auto&& derived = static_cast(*this); - if(derived.empty()) - MIGRAPHX_THROW("Visiting empty data!"); - auto&& s = derived.get_shape(); - s.visit_type([&](auto as) { v(make_view(s, as.from(derived.data()))); }, - [&] { tv(derived.get_sub_objects()); }); - } - - /** - * @brief Visits the data - * - * This will call the visitor function with a `tensor_view` based on the shape of the data. - * - * @param v A function to be called with `tensor_view` - */ - template - void visit(Visitor v) const - { - visit(v, [&](const auto&) { MIGRAPHX_THROW("Invalid tuple type"); }); - } - - /// Returns true if the raw data is only one element - bool single() const - { - auto&& s = static_cast(*this).get_shape(); - return s.elements() == 1; - } - - /** - * @brief Retrieves a single element of data - * - * @param n The index to retrieve the data from - * @tparam T The type of data to be retrieved - * @return The element as `T` - */ - template - T at(std::size_t n = 0) const - { - T result; - this->visit_at([&](auto x) { result = x; }, n); - return result; - } - - struct auto_cast - { - const Derived* self; - template - operator T() - { - assert(self->single()); - return self->template at(); - } - - template - using is_data_ptr = - bool_c<(std::is_void{} or std::is_same>{} or - std::is_same>{})>; - - template - using get_data_type = std::conditional_t{}, float, T>; - - template - bool matches() const - { - return is_data_ptr{} or - self->get_shape().type() == migraphx::shape::get_type>{}; - } - - template - operator T*() - { - using type = std::remove_cv_t; - assert(matches()); - return reinterpret_cast(self->data()); - } - }; - - /// Implicit conversion of raw data pointer - auto_cast implicit() const { return {static_cast(this)}; } - - /// Get a tensor_view to the data - template - tensor_view get() const - { - auto&& s = static_cast(*this).get_shape(); - auto&& buffer = static_cast(*this).data(); - if(s.type() != migraphx::shape::get_type{}) - MIGRAPHX_THROW("Incorrect data type for raw data"); - return make_view(s, reinterpret_cast(buffer)); - } - - /// Cast the data pointer - template - T* cast() const - { - auto&& buffer = static_cast(*this).data(); - assert(static_cast(*this).get_shape().type() == - migraphx::shape::get_type{}); - return reinterpret_cast(buffer); - } - - std::string to_string() const - { - std::stringstream ss; - ss.precision(std::numeric_limits::max_digits10); - ss << static_cast(*this); - return ss.str(); - } - - template - std::vector to_vector() const - { - std::vector result(static_cast(*this).get_shape().elements()); - this->visit([&](auto x) { result.assign(x.begin(), x.end()); }); - return result; - } -}; - -namespace detail { -template -void visit_all_flatten(const shape& s, V1&& v1, V2&& v2, Ts&&... xs) -{ - s.visit_type([&](auto as) { v1(make_view(xs.get_shape(), as.from(xs.data()))...); }, - [&] { v2(xs.get_sub_objects()...); }); -} - -template -auto visit_all_pack(const shape& s, V1&& v1, V2&& v2) -{ - return [&](auto&&... xs) { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70100 - visit_all_flatten(s, v1, v2, xs...); - }; -} - -template -auto visit_all_pack(const shape& s, V1&& v1) -{ - return visit_all_pack(s, v1, [](auto&&...) { MIGRAPHX_THROW("Invalid tuple type"); }); -} -} // namespace detail - -/** - * @brief Visits every object together - * @details This will visit every object, but assumes each object is the same type. This can reduce - * the deeply nested visit calls. Returns a function that takes the visitor callback. - * Calling syntax is `visit_all(xs...)([](auto... ys) {})` where `xs...` and `ys...` are the - * same number of parameters. - * - * @param x A raw data object - * @param xs Many raw data objects. - * @return A function to be called with the visitor - */ -template -auto visit_all(T&& x, Ts&&... xs) -{ - auto&& s = x.get_shape(); - std::initializer_list types = {xs.get_shape().type()...}; - if(not std::all_of(types.begin(), types.end(), [&](shape::type_t t) { return t == s.type(); })) - MIGRAPHX_THROW("Types must be the same"); - return [&](auto... vs) { detail::visit_all_pack(s, vs...)(x, xs...); }; -} - -/** - * @brief Visits every object together - * @details This will visit every object, but assumes each object is the same type. This can reduce - * the deeply nested visit calls. Returns a function that takes the visitor callback. - * - * @param x A vector of raw data objects. Types must all be the same. - * @return A function to be called with the visitor - */ -template -auto visit_all(const std::vector& x) -{ - auto&& s = x.front().get_shape(); - if(not std::all_of( - x.begin(), x.end(), [&](const T& y) { return y.get_shape().type() == s.type(); })) - MIGRAPHX_THROW("Types must be the same"); - return [&](auto v) { - s.visit_type([&](auto as) { - using type = typename decltype(as)::type; - std::vector> result; - std::transform(x.begin(), x.end(), std::back_inserter(result), [&](const auto& y) { - return make_view(y.get_shape(), as.from(y.data())); - }); - v(result); - }); - }; -} - -template {} and - std::is_base_of{})> -bool operator==(const T& x, const U& y) -{ - auto&& xshape = x.get_shape(); - auto&& yshape = y.get_shape(); - bool result = x.empty() and y.empty(); - if(not result and xshape == yshape) - { - visit_all(x, y)([&](auto xview, auto yview) { result = xview == yview; }, - [&](auto&& xs, auto&& ys) { - result = std::equal(xs.begin(), xs.end(), ys.begin(), ys.end()); - }); - } - return result; -} - -template {} and - std::is_base_of{})> -bool operator!=(const T& x, const U& y) -{ - return not(x == y); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/reduce_dims.hpp b/docker/rocm/migraphx/include/migraphx/reduce_dims.hpp deleted file mode 100644 index 266d817b6..000000000 --- a/docker/rocm/migraphx/include/migraphx/reduce_dims.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REDUCE_DIMS_HPP -#define MIGRAPHX_GUARD_RTGLIB_REDUCE_DIMS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT std::vector reduce_dims(const std::vector& shapes); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/reflect.hpp b/docker/rocm/migraphx/include/migraphx/reflect.hpp deleted file mode 100644 index ff13eb617..000000000 --- a/docker/rocm/migraphx/include/migraphx/reflect.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REFLECT_HPP -#define MIGRAPHX_GUARD_RTGLIB_REFLECT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace detail { - -struct reflect_placeholder -{ - template - int operator()(Ts&&...) const - { - return 0; - } -}; - -template -auto reflect_impl(rank<1>, T& x, Selector f) -> decltype(T::reflect(x, f)) -{ - return T::reflect(x, std::move(f)); -} - -template -auto reflect_impl(rank<0>, T&, Selector) -{ - return pack(); -} - -template -auto reflectable_impl(rank<1>, const T& x) - -> decltype(T::reflect(x, reflect_placeholder{}), std::true_type{}); - -template -auto reflectable_impl(rank<0>, const T&) -> decltype(std::false_type{}); - -template -struct remove_rvalue_reference -{ - using type = T; -}; - -template -struct remove_rvalue_reference -{ - using type = T; -}; - -template -struct wrapper -{ - using type = typename remove_rvalue_reference::type; - type data; // NOLINT - type get() const { return data; } -}; - -template -wrapper wrap(std::remove_reference_t& x) -{ - return wrapper{std::forward(x)}; -} - -template -using auto_tuple_t = std::tuple::type...>; - -template -auto_tuple_t auto_tuple(Ts&&... xs) -{ - return auto_tuple_t{std::forward(xs)...}; -} - -} // namespace detail - -template -using is_reflectable = decltype(detail::reflectable_impl(rank<1>{}, std::declval())); - -template -auto reflect(T& x, Selector f) -{ - return detail::reflect_impl(rank<1>{}, x, std::move(f)); -} - -template -auto reflect_tie(T& x) -{ - return reflect(x, [](auto&& y, auto&&...) { - // cppcheck-suppress UnnecessaryElseStatement - if constexpr(is_reflectable{}) - { - auto t = reflect_tie(y); - return detail::wrap(t); - } - else - { - return detail::wrap(y); - } - })([](auto&&... xs) { return detail::auto_tuple(xs.get()...); }); -} - -template -void reflect_each(T& x, F f) -{ - return reflect(x, [](auto&& y, auto... ys) { - return pack(detail::wrap(y), ys...); - })([&](auto&&... xs) { - each_args([&](auto p) { p([&](auto&& y, auto... ys) { f(y.get(), ys...); }); }, xs...); - }); -} - -template -struct reflect_equality -{ - friend bool operator==(const T& x, const T& y) { return reflect_tie(x) == reflect_tie(y); } - friend bool operator!=(const T& x, const T& y) { return not(x == y); } -}; - -template -struct reflect_stream -{ - template - friend Stream& operator<<(Stream& os, const T& x) - { - char d = '{'; - reflect_each(x, [&](const auto& y, const auto& name) { - os << d << name << "=" << y; - d = ','; - }); - os << "}"; - return os; - } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/register_op.hpp b/docker/rocm/migraphx/include/migraphx/register_op.hpp deleted file mode 100644 index c3bfb32e5..000000000 --- a/docker/rocm/migraphx/include/migraphx/register_op.hpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REGISTER_OP_HPP -#define MIGRAPHX_GUARD_RTGLIB_REGISTER_OP_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// unregister all ops for specified target, useful when unloading dynamically plugged-in target lib -MIGRAPHX_EXPORT void unregister_op(const std::string& op_name); - -namespace detail { -struct op_handler -{ - operation op; - std::string name; - op_handler(const operation& op_r) : op(op_r), name(op.name()){}; - ~op_handler() { unregister_op(name); } -}; - -} // namespace detail - -MIGRAPHX_EXPORT void register_op_init(); - -MIGRAPHX_EXPORT void register_op(const operation& op); - -MIGRAPHX_EXPORT operation load_op(const std::string& name); - -MIGRAPHX_EXPORT bool has_op(const std::string& name); - -MIGRAPHX_EXPORT std::vector get_operators(); - -template -void register_op() -{ - register_op_init(); // instantiate static op_map; - static auto op_h = detail::op_handler(T{}); - register_op(op_h.op); -} - -struct register_op_action -{ - template - static void apply() - { - register_op(); - } -}; - -template -using auto_register_op = auto_register; - -#define MIGRAPHX_REGISTER_OP(...) MIGRAPHX_AUTO_REGISTER(register_op_action, __VA_ARGS__) - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/register_target.hpp b/docker/rocm/migraphx/include/migraphx/register_target.hpp deleted file mode 100644 index 1d274433e..000000000 --- a/docker/rocm/migraphx/include/migraphx/register_target.hpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REGISTER_TARGET_HPP -#define MIGRAPHX_GUARD_RTGLIB_REGISTER_TARGET_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT void register_target_init(); -MIGRAPHX_EXPORT void register_target(const target& t); -MIGRAPHX_EXPORT void unregister_target(const std::string& name); -MIGRAPHX_EXPORT target make_target(const std::string& name); -MIGRAPHX_EXPORT std::vector get_targets(); - -namespace detail { -struct target_handler -{ - target t; - std::string target_name; - explicit target_handler(target t_r) : t(std::move(t_r)), target_name(t.name()) {} - ~target_handler() { unregister_target(target_name); } -}; -} // namespace detail - -template -void register_target() -{ - register_target_init(); - static auto t_h = detail::target_handler(T{}); - register_target(t_h.t); -} - -struct register_target_action -{ - template - static void apply() - { - register_target(); - } -}; - -template -using auto_register_target = auto_register; - -#define MIGRAPHX_REGISTER_TARGET(...) MIGRAPHX_AUTO_REGISTER(register_target_action, __VA_ARGS__) - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/replace_allocate.hpp b/docker/rocm/migraphx/include/migraphx/replace_allocate.hpp deleted file mode 100644 index 2eabea50b..000000000 --- a/docker/rocm/migraphx/include/migraphx/replace_allocate.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REPLACE_ALLOCATE_HPP -#define MIGRAPHX_GUARD_RTGLIB_REPLACE_ALLOCATE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -/** - * Replace `allocate` instructions with target allocations or output parameters. - */ -struct MIGRAPHX_EXPORT replace_allocate -{ - allocation_model model; - bool offload_copy = false; - std::string name() const { return "replace_allocate"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/requires.hpp b/docker/rocm/migraphx/include/migraphx/requires.hpp deleted file mode 100644 index 8b5e333b5..000000000 --- a/docker/rocm/migraphx/include/migraphx/requires.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_REQUIRES_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_REQUIRES_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct and_ : std::is_same, and_<(Bs or true)...>> // NOLINT -{ -}; - -template -using bool_c = std::integral_constant; - -#define MIGRAPHX_REQUIRES_PRIMITIVE_CAT(x, y) x##y -#define MIGRAPHX_REQUIRES_CAT(x, y) MIGRAPHX_REQUIRES_PRIMITIVE_CAT(x, y) - -#define MIGRAPHX_REQUIRES_VAR() MIGRAPHX_REQUIRES_CAT(PrivateRequires, __LINE__) - -#ifdef CPPCHECK -#define MIGRAPHX_REQUIRES(...) class = void -#define MIGRAPHX_CLASS_REQUIRES(...) void -#else -#define MIGRAPHX_REQUIRES(...) \ - long MIGRAPHX_REQUIRES_VAR() = __LINE__, \ - typename std::enable_if<(MIGRAPHX_REQUIRES_VAR() == __LINE__ and \ - (migraphx::and_<__VA_ARGS__>{})), \ - int>::type = 0 -#define MIGRAPHX_CLASS_REQUIRES(...) typename std::enable_if<(migraphx::and_<__VA_ARGS__>{})>::type -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_gelu.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_gelu.hpp deleted file mode 100644 index 6db8f1ee2..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_gelu.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REWRITE_GELU_HPP -#define MIGRAPHX_GUARD_RTGLIB_REWRITE_GELU_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Rewrite GELU blocks as different approximations. - */ -struct MIGRAPHX_EXPORT rewrite_gelu -{ - bool fast_math = true; - std::string name() const { return "rewrite_gelu"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_low_precision.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_low_precision.hpp deleted file mode 100644 index 3529451be..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_low_precision.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REWRITE_LOW_PRECISION_HPP -#define MIGRAPHX_GUARD_RTGLIB_REWRITE_LOW_PRECISION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Rewrite operators in low precision types to avoid going out of precision bounds. - */ -struct MIGRAPHX_EXPORT rewrite_low_precision -{ - std::string name() const { return "rewrite_low_precision"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_pooling.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_pooling.hpp deleted file mode 100644 index ebef98347..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_pooling.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REWRITE_POOLING_HPP -#define MIGRAPHX_GUARD_RTGLIB_REWRITE_POOLING_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Rewrite pooling to reduce_mean - */ -struct MIGRAPHX_EXPORT rewrite_pooling -{ - std::string name() const { return "rewrite_pooling"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_quantization.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_quantization.hpp deleted file mode 100644 index d4e33a373..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_quantization.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REWRITE_QUANTIZATION_HPP -#define MIGRAPHX_GUARD_RTGLIB_REWRITE_QUANTIZATION_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Rewrite quantization ops to equivalent operators - */ -struct MIGRAPHX_EXPORT rewrite_quantization -{ - std::string name() const { return "rewrite_quantization"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_reduce.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_reduce.hpp deleted file mode 100644 index 481bc7581..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_reduce.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_SRC_INCLUDE_MIGRAPHX_REWRITE_REDUCE -#define MIGRAPHX_GUARD_SRC_INCLUDE_MIGRAPHX_REWRITE_REDUCE - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -struct MIGRAPHX_EXPORT rewrite_reduce -{ - std::string name() const { return "rewrite_reduce"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_reshapes.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_reshapes.hpp deleted file mode 100644 index c064091c2..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_reshapes.hpp +++ /dev/null @@ -1,244 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_REWRITE_RESHAPES_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_REWRITE_RESHAPES_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct rewrite_reshapes_base -{ - template - static instruction_ref insert(module_pass_manager& mpm, - instruction_ref ins, - const std::vector& inputs, - const AxesMap&) - { - return mpm.get_module().insert_instruction( - ins, ins->get_operator(), inputs, ins->module_inputs()); - } - - template - static bool supports(instruction_ref, std::vector&, const AxesMap&) - { - return true; - } - - static std::vector base_dims(instruction_ref ins) - { - return ins->get_shape().lens(); - } -}; - -template -struct rewrite_reshapes -{ - std::string name() const { return "rewrite_reshapes"; } - struct find_op_reshape_op - { - std::string op1; - std::string op2; - - auto matcher() const - { - auto reshapes = match::name("reshape", - "squeeze", - "unsqueeze", - "flatten", - "transpose", - "contiguous", - "multibroadcast", - "broadcast")(match::used_once()); - auto pointwise = match::name(op1)(match::used_once()); - auto reshapes_pointwise = - reshapes(match::arg(0)(match::skip(reshapes())(pointwise.bind("x")))); - return match::name(op2)( - match::any_of[match::inputs()](reshapes_pointwise.bind("input"))); - } - - template - static instruction_ref find_input_if(instruction_ref start, instruction_ref last, F f) - { - while(start != last) - { - if(f(start)) - return start; - if(start->inputs().size() != 1) - return last; - start = start->inputs().front(); - } - return last; - } - - template - static bool any_input_of(instruction_ref start, instruction_ref last, F f) - { - return find_input_if(start, last, f) != last; - } - - static bool match_input(instruction_ref ins, instruction_ref x_ins) - { - if(ins->inputs().empty()) - return false; - auto input = ins->inputs().front(); - if(input->name() == "contiguous") - return match_input(input, x_ins); - return x_ins == input; - } - - static std::optional is_broadcasted(instruction_ref start, instruction_ref last) - { - auto broadcast_ins = - find_input_if(start, last, [&](auto i) { return i->name() == "multibroadcast"; }); - bool result = broadcast_ins != last; - if(result and not match_input(broadcast_ins, last)) - return nullopt; - return result; - } - - static bool is_broadcast(instruction_ref ins) { return ins->name() == "multibroadcast"; } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - auto input_ins = r.instructions["input"]; - - // If its just a broadcast then skip - if(not any_input_of(input_ins, x_ins, [](instruction_ref x) { - return not contains({"multibroadcast", "broadcast", "contiguous"}, x->name()); - })) - return; - - auto dims1 = T::base_dims(ins); - auto dims2 = T::base_dims(x_ins); - - if(elements(dims1) != elements(dims2)) - return; - - std::vector ops; - auto next_ins = input_ins; - while(next_ins != x_ins) - { - ops.push_back(next_ins->get_operator()); - next_ins = next_ins->inputs().front(); - } - assert(next_ins == x_ins); - std::reverse(ops.begin(), ops.end()); - - auto desc = - shape_transform_descriptor::create(x_ins->get_shape().lens(), ops).rebase(dims2); - if(desc.empty()) - return; - auto cdims = desc.common_dims(); - auto reshape_input = [&](const auto& ins_to_insert, auto generate) { - return [&, generate](auto input) { - auto gops = std::invoke(generate, desc, input->get_shape().lens()); - auto start = input; - for(const auto& op : gops) - { - start = mpm.get_module().insert_instruction(ins_to_insert, op, start); - } - return start; - }; - }; - auto x_inputs = x_ins->inputs(); - std::transform( - x_inputs.begin(), - x_inputs.end(), - x_inputs.begin(), - reshape_input(x_ins, &shape_transform_descriptor::generate_common_from_src)); - auto new_x_ins = insert(mpm, x_ins, x_inputs, desc.common_axes_map_from_src()); - if(new_x_ins->get_shape().lens() != cdims) - { - new_x_ins = mpm.get_module().insert_instruction( - x_ins, make_op("multibroadcast", {{"out_lens", cdims}}), new_x_ins); - } - - auto inputs = ins->inputs(); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](auto input) { - if(input == input_ins) - return new_x_ins; - return reshape_input(ins, - &shape_transform_descriptor::generate_common_from_dst)(input); - }); - auto pw = insert(mpm, ins, inputs, desc.common_axes_map_from_dst()); - auto rins = - reshape_input(ins, &shape_transform_descriptor::generate_dst_from_common)(pw); - mpm.get_module().replace_instruction(ins, rins); - } - - static bool same_dims(instruction_ref ins) - { - return all_of(ins->inputs(), [&](auto input) { - return input->get_shape().lens() == ins->get_shape().lens(); - }); - } - - template - static instruction_ref insert(module_pass_manager& mpm, - instruction_ref ins, - const std::vector& inputs, - const AxesMap& am) - { - if(ins->name() == "pointwise") - return mpm.get_module().insert_instruction( - ins, ins->get_operator(), inputs, ins->module_inputs()); - return T::insert(mpm, ins, inputs, am); - } - }; - - void apply(module_pass_manager& mpm) const - { - if(T::name() == "pointwise") - { - match::find_matches(mpm, find_op_reshape_op{"pointwise", T::name()}); - } - else - { - match::find_matches(mpm, - find_op_reshape_op{"pointwise", T::name()}, - find_op_reshape_op{T::name(), "pointwise"}, - find_op_reshape_op{T::name(), T::name()}); - } - mpm.run_pass(simplify_reshapes{1}); - mpm.run_pass(eliminate_common_subexpression{}); - mpm.run_pass(dead_code_elimination{}); - } -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_REWRITE_RESHAPES_HPP diff --git a/docker/rocm/migraphx/include/migraphx/rewrite_rnn.hpp b/docker/rocm/migraphx/include/migraphx/rewrite_rnn.hpp deleted file mode 100644 index 61524d083..000000000 --- a/docker/rocm/migraphx/include/migraphx/rewrite_rnn.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REWRITE_RNN_HPP -#define MIGRAPHX_GUARD_RTGLIB_REWRITE_RNN_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Rewrite rnn to gemm and add. - */ -struct MIGRAPHX_EXPORT rewrite_rnn -{ - std::string name() const { return "rewrite_rnn"; } - void apply(module& m) const; - - private: - // for vanilla rnn operators - void apply_vanilla_rnn(module& m, instruction_ref ins) const; - std::vector vanilla_rnn_cell(bool is_forward, - module& m, - instruction_ref ins, - std::vector inputs, - const operation& actv_func) const; - std::vector vanilla_rnn_actv_funcs(instruction_ref ins) const; - - // for gru operators - void apply_gru(module& m, instruction_ref ins) const; - std::vector gru_cell(bool is_forward, - module& m, - instruction_ref ins, - std::vector inputs, - int linear_before_reset, - const operation& actv_func1, - const operation& actv_func2) const; - - std::vector gru_actv_funcs(instruction_ref ins) const; - - // for lstm operators - void apply_lstm(module& m, instruction_ref ins) const; - std::vector lstm_cell(bool is_forward, - module& m, - instruction_ref ins, - std::vector inputs, - const operation& actv_func1, - const operation& actv_func2, - const operation& actv_func3) const; - - std::vector lstm_actv_funcs(instruction_ref ins) const; - - bool is_variable_seq_lens(const module& m, instruction_ref seq_lens) const; - instruction_ref replace_last_hs_output(module& m, - instruction_ref ins, - instruction_ref seq_lens, - instruction_ref last_hs_output, - op::rnn_direction dirct) const; - - void replace_last_cell_output(module& m, - instruction_ref ins, - instruction_ref seq_lens, - instruction_ref cell_outputs, - instruction_ref last_cell_output, - op::rnn_direction dirct) const; - - std::size_t get_seq_len(const module& m, instruction_ref input, instruction_ref seq_lens) const; - - instruction_ref pad_hidden_states(module& m, - instruction_ref seq, - instruction_ref seq_lens, - instruction_ref hs) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/run_loop.hpp b/docker/rocm/migraphx/include/migraphx/run_loop.hpp deleted file mode 100644 index e12b82fd2..000000000 --- a/docker/rocm/migraphx/include/migraphx/run_loop.hpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_RUN_LOOP_HPP -#define MIGRAPHX_GUARD_RTGLIB_RUN_LOOP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -argument run_loop(const LoopModel& model, - const std::vector& scan_output_directions, - T& ctx, - std::vector args, - const std::vector& mods, - const std::function( - module_ref&, const std::unordered_map&)>& run) -{ - std::vector> results; - // process argu lists - auto iter_num = args.at(0).at(); - auto cond = args.at(1).at(); - - auto input_num = (args.size() - 2) / 2; - auto dep_num = input_num - 2; - - module_ref mod = mods.at(0); - auto param_name_shapes = mod->get_parameter_shapes(); - auto param_names = mod->get_parameter_names(); - - std::vector dep0(args.begin() + input_num + 1, args.begin() + 2 * input_num); - std::vector dep1(args.begin() + 2 * input_num, args.begin() + 2 * input_num + 1); - auto ins_outputs = args.back().get_sub_objects(); - dep1.insert(dep1.end(), ins_outputs.begin(), ins_outputs.begin() + dep_num); - std::array, 2> loop_carry_deps = {dep0, dep1}; - - // loop iter argument - std::vector in_args = {args.at(input_num), dep1.at(0)}; - in_args.insert(in_args.end(), args.begin() + 2, args.begin() + input_num); - - std::vector out_args = dep0; - out_args.insert(out_args.end(), ins_outputs.begin() + dep_num, ins_outputs.end()); - std::vector scan_outputs(ins_outputs.begin() + dep_num, ins_outputs.end()); - - auto out_param_indices = model.get_output_params(*mod); - - int64_t iter = 0; - for(iter = 0; iter < iter_num and cond; ++iter) - { - // copy iter num and cond to device memory - model.copy(ctx, iter, in_args.at(0)); - model.copy(ctx, cond, in_args.at(1)); - - // wrap up the inputs and outputs - std::unordered_map params; - int input_index = 0; - for(const auto& name : param_names) - { - auto ps = mod->get_parameter_shape(name); - if(ps == shape{}) - { - continue; - } - - // it is an input parameter - if(not contains(out_param_indices, name)) - { - params[name] = in_args.at(input_index++); - } - else - { - auto output_index = out_param_indices[name]; - if(output_index > dep_num) - { - int64_t dir = scan_output_directions.empty() - ? 0 - : scan_output_directions[output_index - dep_num - 1]; - auto idx = (1 - dir) * iter + dir * (iter_num - 1 - iter); - const auto& arg = out_args.at(output_index); - assert((idx + 1) * ps.bytes() <= arg.get_shape().bytes()); - params[name] = argument(ps, arg.data() + idx * ps.bytes()); - } - else - { - params[name] = out_args.at(output_index); - } - } - } - - auto mod_args = run(mod, params); - - // copy back cond to be used next iteration - model.copy(ctx, mod_args.at(0), cond); - - // mod outputs are used as next loop input - std::copy(mod_args.begin(), mod_args.begin() + dep_num + 1, in_args.begin() + 1); - const auto& dep_out = loop_carry_deps[(iter + 1) % 2]; - std::copy(dep_out.begin(), dep_out.end(), out_args.begin()); - - std::vector mod_scan_outs(mod_args.begin() + 1 + dep_num, mod_args.end()); - model.append(mod_scan_outs, scan_outputs, scan_output_directions, iter, iter_num); - } - - out_args.erase(out_args.begin()); - std::copy(in_args.begin() + 2, in_args.end(), out_args.begin()); - model.set_zero(ctx, scan_outputs, iter); - - return {out_args}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/sat_ops.hpp b/docker/rocm/migraphx/include/migraphx/sat_ops.hpp deleted file mode 100644 index 0ccff33b7..000000000 --- a/docker/rocm/migraphx/include/migraphx/sat_ops.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SAT_OPS_HPP -#define MIGRAPHX_GUARD_RTGLIB_SAT_OPS_HPP - -#include -#include - -template -constexpr T mul_sat(T a, T b) noexcept -{ - T c; - if(not __builtin_mul_overflow(a, b, &c)) - { - return c; - } - if constexpr(std::is_unsigned{}) - { - return std::numeric_limits::max(); - } - else if(a < 0 != b < 0) - { - return std::numeric_limits::min(); - } - return std::numeric_limits::max(); -} - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/schedule.hpp b/docker/rocm/migraphx/include/migraphx/schedule.hpp deleted file mode 100644 index 3f0b377f2..000000000 --- a/docker/rocm/migraphx/include/migraphx/schedule.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SCHEDULE_HPP -#define MIGRAPHX_GUARD_RTGLIB_SCHEDULE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Schedule instructions for concurrent execution - */ -struct MIGRAPHX_EXPORT schedule -{ - schedule_model model{}; - bool enable = true; - std::string name() const { return "schedule"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/schedule_model.hpp b/docker/rocm/migraphx/include/migraphx/schedule_model.hpp deleted file mode 100644 index a27f5379f..000000000 --- a/docker/rocm/migraphx/include/migraphx/schedule_model.hpp +++ /dev/null @@ -1,364 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_SCHEDULE_MODEL_HPP -#define MIGRAPHX_GUARD_SCHEDULE_MODEL_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -struct operation; - -#ifdef DOXYGEN - -/// An interface for target-dependent model for the scheduler -struct schedule_model -{ - /// Get the number of concurrent instruction allowed - std::size_t concurrency() const; - /// Schedule a concurrent instruction - void sched(module& m, instruction_ref ins, std::size_t n) const; - // Insert necessary waits before an instruction - void wait(module& m, instruction_ref ins, std::size_t wait_id) const; - // Insert necessary records after an instruction - void record(module& m, instruction_ref ins, std::size_t wait_id) const; - /// Compute weights for an operation - std::size_t weight(const operation& op) const; -}; - -#else - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT schedule_model -{ - // - std::size_t concurrency() const; - // - void sched(module& m, instruction_ref ins, std::size_t n) const; - // - void wait(module& m, instruction_ref ins, std::size_t wait_id) const; - // - void record(module& m, instruction_ref ins, std::size_t wait_id) const; - // - std::size_t weight(const operation& op) const; -}; - -#else - -struct schedule_model -{ - private: - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().concurrency(), - std::declval().sched(std::declval(), - std::declval(), - std::declval()), - std::declval().wait(std::declval(), - std::declval(), - std::declval()), - std::declval().record(std::declval(), - std::declval(), - std::declval()), - std::declval().weight(std::declval()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - schedule_model() = default; - - template < - typename PrivateDetailTypeErasedT, - typename = private_te_constraints, - typename = typename std::enable_if< - not std::is_same, schedule_model>{}>::type> - schedule_model(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template < - typename PrivateDetailTypeErasedT, - typename = private_te_constraints, - typename = typename std::enable_if< - not std::is_same, schedule_model>{}>::type> - schedule_model& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - schedule_model rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::size_t concurrency() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().concurrency(); - } - - void sched(module& m, instruction_ref ins, std::size_t n) const - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().sched(m, ins, n); - } - - void wait(module& m, instruction_ref ins, std::size_t wait_id) const - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().wait(m, ins, wait_id); - } - - void record(module& m, instruction_ref ins, std::size_t wait_id) const - { - assert((*this).private_detail_te_handle_mem_var); - (*this).private_detail_te_get_handle().record(m, ins, wait_id); - } - - std::size_t weight(const operation& op) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().weight(op); - } - - friend bool is_shared(const schedule_model& private_detail_x, - const schedule_model& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::size_t concurrency() const = 0; - virtual void sched(module& m, instruction_ref ins, std::size_t n) const = 0; - virtual void wait(module& m, instruction_ref ins, std::size_t wait_id) const = 0; - virtual void record(module& m, instruction_ref ins, std::size_t wait_id) const = 0; - virtual std::size_t weight(const operation& op) const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::size_t concurrency() const override { return private_detail_te_value.concurrency(); } - - void sched(module& m, instruction_ref ins, std::size_t n) const override - { - - private_detail_te_value.sched(m, ins, n); - } - - void wait(module& m, instruction_ref ins, std::size_t wait_id) const override - { - - private_detail_te_value.wait(m, ins, wait_id); - } - - void record(module& m, instruction_ref ins, std::size_t wait_id) const override - { - - private_detail_te_value.record(m, ins, wait_id); - } - - std::size_t weight(const operation& op) const override - { - - return private_detail_te_value.weight(op); - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const schedule_model* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(schedule_model* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(schedule_model& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const schedule_model& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/serialize.hpp b/docker/rocm/migraphx/include/migraphx/serialize.hpp deleted file mode 100644 index b3525e41f..000000000 --- a/docker/rocm/migraphx/include/migraphx/serialize.hpp +++ /dev/null @@ -1,284 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SERIALIZE_HPP -#define MIGRAPHX_GUARD_RTGLIB_SERIALIZE_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// Avoid implicit conversion with ADL lookup -template -void migraphx_to_value(value&, const T&) = delete; - -template -value to_value(const T& x); - -template -void from_value(const value& v, T& x); - -template -T from_value(const value& v) -{ - T x{}; - from_value(v, x); - return x; -} - -namespace detail { - -template {})> -value to_value_impl(rank<0>, const T&) -{ - return value::object{}; -} - -template -auto to_value_impl(rank<1>, const T& x) -> decltype(std::tuple_size{}, value{}) -{ - value result = value::array{}; - repeat_c{}>([&](auto i) { result.push_back(to_value(std::get(x))); }); - return result; -} - -template -auto to_value_impl(rank<2>, const T& x) -> decltype(x.begin(), x.end(), value{}) -{ - value result = value::array{}; - for(auto&& y : x) - { - result.insert(to_value(y)); - } - return result; -} - -template {})> -value to_value_impl(rank<3>, const T& x) -{ - value result = value::object{}; - reflect_each(x, [&](auto&& y, std::string name) { result.emplace(name, to_value(y)); }); - return result; -} - -template -auto to_value_impl(rank<4>, const optional& x) -{ - value result{}; - if(x.has_value()) - return to_value(*x); - return result; -} - -template {})> -value to_value_impl(rank<5>, const T& x) -{ - return std::int64_t{x}; -} - -template {})> -value to_value_impl(rank<6>, const T& x) -{ - return std::uint64_t{x}; -} - -template {})> -value to_value_impl(rank<7>, const T& x) -{ - return double{x}; -} - -template {})> -value to_value_impl(rank<8>, const T& x) -{ - return x; -} - -inline value to_value_impl(rank<9>, const std::string& x) { return x; } - -template -auto to_value_impl(rank<10>, const T& x) -> decltype(migraphx_to_value(x)) -{ - return migraphx_to_value(x); -} - -template -auto to_value_impl(rank<11>, const T& x) -> decltype(x.to_value()) -{ - return x.to_value(); -} - -template -auto to_value_impl(rank<12>, const T& x) - -> decltype(migraphx_to_value(std::declval(), x), value{}) -{ - value v; - migraphx_to_value(v, x); - return v; -} - -template {})> -value to_value_impl(rank<13>, const T& x) -{ - return x; -} - -template {})> -void from_value_impl(rank<0>, const value& v, T& x) -{ - if(not v.is_object()) - MIGRAPHX_THROW("Expected an object"); - if(not v.get_object().empty()) - MIGRAPHX_THROW("Expected an empty object"); - x = T{}; -} - -template -auto from_value_impl(rank<1>, const value& v, T& x) -> decltype(std::tuple_size{}, void()) -{ - repeat_c{}>( - [&](auto i) { std::get(x) = from_value>(v[i]); }); -} - -template -auto from_value_impl(rank<2>, const value& v, T& x) - -> decltype(x.insert(x.end(), *x.begin()), void()) -{ - x.clear(); - for(auto&& e : v) - x.insert(x.end(), from_value(e)); -} - -template {})> -auto from_value_impl(rank<3>, const value& v, T& x) - -> decltype(x.insert(x.end(), *x.begin()), void()) -{ - x.clear(); - if(v.is_binary()) - { - for(auto&& e : v.get_binary()) - x.insert(x.end(), e); - } - else - { - for(auto&& e : v) - x.insert(x.end(), from_value(e)); - } -} - -template -auto from_value_impl(rank<4>, const value& v, T& x) - -> decltype(x.insert(*x.begin()), std::declval(), void()) -{ - if(v.is_object()) - { - x.clear(); - for(auto&& e : v) - x.emplace(from_value(e.get_key()), - from_value(e)); - } - else if(v.is_array()) - { - x.clear(); - for(auto&& e : v) - { - if(e.size() != 2) - MIGRAPHX_THROW("Expected a pair"); - x.emplace(from_value(e[0]), - from_value(e[1])); - } - } - else - { - MIGRAPHX_THROW("Expected object or array"); - } -} - -template {})> -void from_value_impl(rank<5>, const value& v, T& x) -{ - reflect_each(x, [&](auto& y, const std::string& name) { - using type = std::decay_t; - if(v.contains(name)) - y = from_value(v.at(name).without_key()); - }); -} - -template -void from_value_impl(rank<6>, const value& v, optional& x) -{ - if(not v.is_null()) - x = from_value(v); -} - -template {} or std::is_enum{})> -void from_value_impl(rank<7>, const value& v, T& x) -{ - x = v.to(); -} - -inline void from_value_impl(rank<8>, const value& v, std::string& x) { x = v.to(); } - -template -auto from_value_impl(rank<9>, const value& v, T& x) -> decltype(x.from_value(v), void()) -{ - x.from_value(v); -} - -template -auto from_value_impl(rank<10>, const value& v, T& x) -> decltype(migraphx_from_value(v, x), void()) -{ - migraphx_from_value(v, x); -} - -template {})> -void from_value_impl(rank<11>, const value& v, T& x) -{ - x = v; -} - -} // namespace detail - -template -value to_value(const T& x) -{ - return detail::to_value_impl(rank<13>{}, x); -} - -template -void from_value(const value& v, T& x) -{ - detail::from_value_impl(rank<11>{}, v, x); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/shape.hpp b/docker/rocm/migraphx/include/migraphx/shape.hpp deleted file mode 100644 index 65eb04d2b..000000000 --- a/docker/rocm/migraphx/include/migraphx/shape.hpp +++ /dev/null @@ -1,452 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_SHAPE_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_SHAPE_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct value; -struct shape_impl; - -struct MIGRAPHX_EXPORT shape -{ - -// Add new types here -// clang-format off -#define MIGRAPHX_SHAPE_VISIT_TYPES(m) \ - m(bool_type, bool) \ - m(half_type, half) \ - m(float_type, float) \ - m(double_type, double) \ - m(uint8_type, uint8_t) \ - m(int8_type, int8_t) \ - m(uint16_type, uint16_t) \ - m(int16_type, int16_t) \ - m(int32_type, int32_t) \ - m(int64_type, int64_t) \ - m(uint32_type, uint32_t) \ - m(uint64_type, uint64_t) \ - m(fp8e4m3fnuz_type, migraphx::fp8::fp8e4m3fnuz) \ - m(fp8e4m3fn_type, migraphx::fp8::fp8e4m3fn) \ - m(fp8e5m2_type, migraphx::fp8::fp8e5m2) \ - m(bf16_type, bf16) \ - m(fp8e5m2fnuz_type, migraphx::fp8::fp8e5m2fnuz) // clang-format on - -#define MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES(x, t) x, - enum type_t - { - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES) tuple_type - }; -#undef MIGRAPHX_SHAPE_GENERATE_ENUM_TYPES - - template - struct get_type; -#define MIGRAPHX_SHAPE_GENERATE_GET_TYPE(x, t) \ - template \ - struct get_type : std::integral_constant \ - { \ - }; - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_GET_TYPE) -#undef MIGRAPHX_SHAPE_GENERATE_GET_TYPE - - template - struct get_type : get_type - { - }; - - struct MIGRAPHX_EXPORT dynamic_dimension - { - std::size_t min = 0; - std::size_t max = 0; - std::set optimals{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.min, "min"), f(self.max, "max"), f(self.optimals, "optimals")); - } - - bool is_fixed() const; - bool has_optimal() const; - - /** - * Return a dynamic_dimension with the intersection of two dynamic_dimension ranges if - * possible. - */ - std::optional intersection(const dynamic_dimension& other) const - { - auto left = std::max(this->min, other.min); - auto right = std::min(this->max, other.max); - if(left <= right) - { - return dynamic_dimension{left, right}; - } - return nullopt; - } - - MIGRAPHX_EXPORT friend bool operator==(const dynamic_dimension& x, - const dynamic_dimension& y); - MIGRAPHX_EXPORT friend bool operator!=(const dynamic_dimension& x, - const dynamic_dimension& y); - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, - const dynamic_dimension& x); - - // compare to fixed std::size_t dimension - MIGRAPHX_EXPORT friend bool operator==(const dynamic_dimension& x, const std::size_t& y); - MIGRAPHX_EXPORT friend bool operator==(const std::size_t& x, const dynamic_dimension& y); - MIGRAPHX_EXPORT friend bool operator!=(const dynamic_dimension& x, const std::size_t& y); - MIGRAPHX_EXPORT friend bool operator!=(const std::size_t& x, const dynamic_dimension& y); - - // add and subtract fixed std::size_t dimension - dynamic_dimension& operator+=(const std::size_t& x); - dynamic_dimension& operator-=(const std::size_t& x); - MIGRAPHX_EXPORT friend dynamic_dimension operator+(const dynamic_dimension& x, - const std::size_t& y); - MIGRAPHX_EXPORT friend dynamic_dimension operator+(const std::size_t& x, - const dynamic_dimension& y); - MIGRAPHX_EXPORT friend dynamic_dimension operator-(const dynamic_dimension& x, - const std::size_t& y); - }; - - static std::string to_sizes_string(const std::vector& shapes); - - static const std::vector& types(); - - static std::string name(type_t t); - static std::string cpp_type(type_t t); - - static bool is_integral(type_t t); - static bool is_compatible(const shape& actual, const shape& expected); - - static bool is_unsigned(type_t t); - - shape(); - shape(type_t t); - shape(type_t t, std::vector l); - shape(type_t t, std::vector l, std::vector s); - - // Force all calls of the format `shape( type_t, { size_t compatibles } )` to map to - // shape(type_t, std::vector l) - shape(type_t t, std::initializer_list d); - - shape(type_t t, std::vector dims); - - // Construct a dynamic shape from vectors of mins, maxes, and optimals. - // optimals_list is a vector of optimals that corresponds to each min and max. - shape(type_t t, - std::vector mins, - std::vector maxes, - std::vector> optimals_list); - - template - shape(type_t t, const Range& l) : shape(t, std::vector(l.begin(), l.end())) - { - } - - template - shape(type_t t, const Range1& l, const Range2& s) - : shape(t, - std::vector(l.begin(), l.end()), - std::vector(s.begin(), s.end())) - { - } - - explicit shape(const std::vector& subs); - - /** - * Creates an output shape with dimensions equal to the input lengths and strides determined - * by the permutation argument such that find_permutation() of the output shape returns the - * inputted permuation. - * - * 2D example: - * parameters: - * l = [2, 3], perm = [1, 0] - * therefore: - * "original" shape = {lens = [3, 2], strides = [2, 1]} - * output_shape = {lens = [2, 3], strides = [1, 2] - * - * 3D example: - * parameters: - * l = [2, 3, 4], perm = [1, 2, 0] - * therefore: - * "original" shape = {lens = [3, 4, 2], strides = [8, 2, 1]} - * output_shape = {lens = [2, 3, 4], strides = [1, 8, 2]} - */ - static shape - from_permutation(type_t t, const std::vector& l, const std::vector& perm); - - type_t type() const; - const std::vector& lens() const; - const std::vector& strides() const; - - /*! - * The number of dimensions in the shape, either static or dynamic. - * Same as the number of indices required to get a data value. - */ - std::size_t ndim() const; - - /*! - * Return the number of elements in the tensor. - */ - std::size_t elements() const; - - /*! - * Return the number of total bytes used for storage of the tensor data; includes subshapes. - * For dynamic shape, returns the maximum number of bytes presuming a packed shape. - */ - std::size_t bytes() const; - - /*! - * Return the size of the type of the main shape. - * Returns 0 if there are subshapes. - */ - std::size_t type_size() const; - - const std::vector& dyn_dims() const; - - /*! - * Minimum lengths for dynamic shape. - * lens() for static shape. - */ - std::vector min_lens() const; - - /*! - * Maximum lengths for dynamic shape. - * lens() for static shape. - */ - std::vector max_lens() const; - - /*! - * Optimum lengths for dynamic shape. - * Empty for static shape. - */ - std::vector> opt_lens() const; - - /// Map multiple indices to space index - std::size_t index(std::initializer_list l) const; - /// Map multiple indices to space index - std::size_t index(const std::vector& l) const; - - /// Map multiple indices from a range of iterator to a space index - template - std::size_t index(Iterator start, Iterator last) const - { - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: index() called on dynamic shape"); - } - assert(std::distance(start, last) <= this->lens().size()); - assert(this->lens().size() == this->strides().size()); - return std::inner_product(start, last, this->strides().begin(), std::size_t{0}); // NOLINT - } - - /// Map element index to space index - std::size_t index(std::size_t i) const; - - /// Map element index to multi-dimensional index - std::vector multi(std::size_t idx) const; - - /// Map element index to multi-dimensional index and put them them into location provided by - /// pointers - void multi_copy(std::size_t idx, std::size_t* start, const std::size_t* end) const; - - /// Check if a multi-dimensional index is within bounds for the shape. - bool multi_within_bounds(std::vector multi) const; - - /// Returns true if the shape is packed (number of elements and buffer size the same) with - /// no padding - bool packed() const; - - /// Returns true if the shape has been transposed. That is the strides are not in descending - /// order - bool transposed() const; - - /// Returns true if the shape is broadcasting a dimension. That is, one of the strides are zero - bool broadcasted() const; - - /// Returns true if the shape is in its standard format. That is, the shape is both packed and - /// not transposed. - bool standard() const; - - /// Returns true if all strides are equal to 0 (scalar tensor) - bool scalar() const; - - /// Return true if the shape is dynamic - bool dynamic() const; - - /// Return true if this shape or any of the sub_shapes are dynamic - bool any_of_dynamic() const; - - shape normalize_standard() const; - - shape as_standard() const; - - shape with_lens(type_t t, const std::vector& l) const; - shape with_lens(const std::vector& l) const; - - shape with_type(type_t t) const; - - // convert the shape to an equivalent dynamic shape with empty optimals - shape to_dynamic() const; - - // convert the shape to a static one setting any non-fixed dynamic_dimensions to x - shape to_static(std::size_t x) const; - - MIGRAPHX_EXPORT friend bool operator==(const shape& x, const shape& y); - MIGRAPHX_EXPORT friend bool operator!=(const shape& x, const shape& y); - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const shape& x); - - template - struct as - { - using type = std::conditional_t{}, int8_t, T>; - - type max() const { return std::numeric_limits::max(); } - - type min() const { return std::numeric_limits::lowest(); } - - type nan() const { return std::numeric_limits::quiet_NaN(); } - - template - type operator()(U u) const - { - return type(u); - } - - template - type* operator()(U* u) const - { - return static_cast(u); - } - - template - const type* operator()(const U* u) const - { - return static_cast(u); - } - - type operator()() const { return {}; } - - std::size_t size(std::size_t n = 1) const { return sizeof(type) * n; } - - auto is_integral() const { return std::is_integral{}; } - auto is_signed() const { return std::is_signed{}; } - auto is_unsigned() const { return std::is_unsigned{}; } - - template - type* from(U* buffer, std::size_t n = 0) const - { - return reinterpret_cast(buffer) + n; - } - - template - const type* from(const U* buffer, std::size_t n = 0) const - { - return reinterpret_cast(buffer) + n; - } - - type_t type_enum() const { return get_type{}; } - }; - - template - static void visit(type_t t, Visitor v, TupleVisitor tv) - { - switch(t) - { - case tuple_type: { - tv(); - return; - } -#define MIGRAPHX_SHAPE_GENERATE_VISITOR_CASE(x, t) \ - case x: v(as()); return; - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_VISITOR_CASE) -#undef MIGRAPHX_SHAPE_GENERATE_VISITOR_CASE - } - MIGRAPHX_THROW("Unknown type"); - } - - template - static void visit(type_t t, Visitor v) - { - return visit(t, v, [] { MIGRAPHX_THROW("Tuple cannot be visited."); }); - } - - template - void visit_type(Visitors... vs) const - { - visit(this->type(), vs...); - } - - template - static void visit_types(Visitor v) - { -#define MIGRAPHX_SHAPE_GENERATE_VISITOR_ALL(x, t) v(as()); - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_VISITOR_ALL) -#undef MIGRAPHX_SHAPE_GENERATE_VISITOR_ALL - } - - std::string type_string() const; - static type_t parse_type(const std::string& s); - - const std::vector& sub_shapes() const; - - /*! - * Returns the number of elements in the data buffer. - * For a dynamic shape, returns the maximum number of elements of the data buffer and assumes it - * is packed. - * Will clip to the maximum of size_t if overflows for dynamic shapes. - */ - std::size_t element_space() const; - - private: - shape(std::shared_ptr pimpl); - std::shared_ptr impl; -}; - -/// Flatten subshapes to a single vector of non-tuple type of shapes -MIGRAPHX_EXPORT std::vector flatten(const std::vector& shapes); - -MIGRAPHX_EXPORT void migraphx_to_value(value& v, const shape& s); -MIGRAPHX_EXPORT void migraphx_from_value(const value& v, shape& s); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/shape_for_each.hpp b/docker/rocm/migraphx/include/migraphx/shape_for_each.hpp deleted file mode 100644 index fdbf2b3a9..000000000 --- a/docker/rocm/migraphx/include/migraphx/shape_for_each.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_SHAPE_FOR_EACH_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_SHAPE_FOR_EACH_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Iterates the given function over the indices from the shape in order. - */ -template -void shape_for_each(const migraphx::shape& s, F f) -{ - std::vector indices(s.lens().size()); - const auto& index_const_ref = indices; - shape ss{s.type(), s.lens()}; - size_t max = ss.elements(); - for(std::size_t i = 0; i < max; i++) - { - std::transform(ss.strides().begin(), - ss.strides().end(), - ss.lens().begin(), - indices.begin(), - [&](std::size_t stride, std::size_t len) { - assert(len > 0 and stride > 0); - return (i / stride) % len; - }); - if constexpr(std::is_invocable{}) - f(index_const_ref, i); - else - f(index_const_ref); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/shape_transform_descriptor.hpp b/docker/rocm/migraphx/include/migraphx/shape_transform_descriptor.hpp deleted file mode 100644 index dd1ff256b..000000000 --- a/docker/rocm/migraphx/include/migraphx/shape_transform_descriptor.hpp +++ /dev/null @@ -1,159 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_SHAPE_TRANSFORM_DESCRIPTOR_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_SHAPE_TRANSFORM_DESCRIPTOR_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct operation; - -// The shape_transform_descriptor class is data structure to simplify shape -// transformations like reshape, transpose, broadcast, etc. This is made up -// of a collection of dimensions which are a collection of subdimensions. -// -// Each subdimension has an axis and a `len`. The `len` is the length of the -// subdimension. The axis represents the axis the dimension originated. It is -// represented as a vector, the first element represents the axis in the -// original dimension and the additional elements are used when such -// dimension is split. The axis is empty when its a broadcasted dimension, -// and a hidden axis can be set if the dimension is associated with a `1` -// dimension in the original shape. -// -// This will first record shape transformations with the `apply` method. This -// will manipulate this data structure to represent how the transformation -// changes the dimensions. -// -// For example, if we start with an initial dimensions as `[x, y, z]` then -// each dimension will have one subdimension that corresponds to each -// original dimension: `[[x:0]], [[y:1]], [[z:2]]`. -// -// When a transpose is applied we would just permutate the dimensions. -// -// When a reshape that would merge dimensions together then the subdimensions -// are copied to the same subdimension. So if we reshape the dimensions as ` -// [x*y, z]` then it would become `[[x:0], [y:1]], [[z:2]]`. If the reshape -// splits the dimension then the subdimension is copied to each dimension and -// the axis is updated to maintain the order. So a reshape of `[2, x/2, y, -// z]` would become: `[[2:0,0]], [[x/2:0,1]], [[y:1]], [[z:2]]`. -// -// After recording the operators, `simplify` method is used to simplify the -// data structure such as merging adjacent dimension, etc. The `generate` -// method is called to generate the operators need to do this -// transformation. -struct MIGRAPHX_EXPORT shape_transform_descriptor -{ - shape_transform_descriptor() = default; - explicit shape_transform_descriptor(const std::vector& dims); - - static shape_transform_descriptor create(const std::vector& dims, - const std::vector& ops); - - shape_transform_descriptor rebase(const std::vector& dims) const; - - bool apply(const std::vector& ops); - bool apply_reshape(const std::vector& rdims); - bool apply_reshape_impl(const std::vector& rdims); - bool apply_transpose(const std::vector& permutation); - bool apply_broadcast(const std::vector& out_lens, - optional axis = nullopt); - void simplify(); - std::size_t elements() const; - std::vector generate() const; - - bool has_broadcast() const; - void flatten_broadcast(); - - std::vector common_dims(const std::vector& input_dims = {}) const; - std::vector - generate_common_from_src(const std::vector& input_dims = {}) const; - std::vector - generate_common_from_dst(const std::vector& input_dims = {}) const; - std::vector - generate_dst_from_common(const std::vector& input_dims = {}) const; - std::vector> common_axes_map_from_src() const; - std::vector> common_axes_map_from_dst() const; - - bool empty() const; - std::vector lens() const; - - struct MIGRAPHX_EXPORT dimension - { - void simplify(); - std::size_t len() const; - struct MIGRAPHX_EXPORT sub - { - std::size_t len; - std::vector axis = {}; - // The hidden axis is used for broadcasted dimensions. The - // original axis has a length of 1, but this subdimension has a - // length greater then 1, so it cant be directly associated with - // the axis. However, it still needs to accounted for. After we - // generate the broadcast we will set the axis to the hidden - // axis, and then length to 1. - std::vector hidden_axis = {}; - - const std::vector& origin_axis() const; - bool has_hidden_axis() const; - - void add_split_axis(std::size_t i); - - void expose(); - void hide(); - - MIGRAPHX_EXPORT friend bool operator==(const sub& x, const sub& y); - MIGRAPHX_EXPORT friend bool operator!=(const sub& x, const sub& y); - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const sub& x); - }; - - MIGRAPHX_EXPORT friend bool operator==(const dimension& x, const dimension& y); - MIGRAPHX_EXPORT friend bool operator!=(const dimension& x, const dimension& y); - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const dimension& x); - - std::vector subdimensions; - }; - MIGRAPHX_EXPORT friend bool operator==(const shape_transform_descriptor& x, - const shape_transform_descriptor& y); - MIGRAPHX_EXPORT friend bool operator!=(const shape_transform_descriptor& x, - const shape_transform_descriptor& y); - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, - const shape_transform_descriptor& x); - std::vector dimensions; - // Rank of the original dimensions - std::size_t rank = 0; -}; - -MIGRAPHX_EXPORT std::vector -optimize_shape_transforms(const std::vector& dims, const std::vector& ops); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_SHAPE_TRANSFORM_DESCRIPTOR_HPP diff --git a/docker/rocm/migraphx/include/migraphx/simple_par_for.hpp b/docker/rocm/migraphx/include/migraphx/simple_par_for.hpp deleted file mode 100644 index 7c23bd7eb..000000000 --- a/docker/rocm/migraphx/include/migraphx/simple_par_for.hpp +++ /dev/null @@ -1,119 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SIMPLE_PAR_FOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_SIMPLE_PAR_FOR_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct joinable_thread : std::thread -{ - template - joinable_thread(Xs&&... xs) : std::thread(std::forward(xs)...) // NOLINT - { - } - - joinable_thread& operator=(joinable_thread&& other) = default; - joinable_thread(joinable_thread&& other) = default; - - ~joinable_thread() - { - if(this->joinable()) - this->join(); - } -}; - -template -auto thread_invoke(std::size_t i, std::size_t tid, F f) -> decltype(f(i, tid)) -{ - f(i, tid); -} - -template -auto thread_invoke(std::size_t i, std::size_t, F f) -> decltype(f(i)) -{ - f(i); -} - -template -void simple_par_for_impl(std::size_t n, std::size_t threadsize, F f) -{ - if(threadsize <= 1) - { - for(std::size_t i = 0; i < n; i++) - thread_invoke(i, 0, f); - } - else - { - std::vector threads(threadsize); -// Using const here causes gcc 5 to ICE -#if(!defined(__GNUC__) || __GNUC__ != 5) - const -#endif - std::size_t grainsize = std::ceil(static_cast(n) / threads.size()); - - std::size_t work = 0; - std::size_t tid = 0; - std::generate(threads.begin(), threads.end(), [=, &work, &tid] { - auto result = joinable_thread([=] { - std::size_t start = work; - std::size_t last = std::min(n, work + grainsize); - for(std::size_t i = start; i < last; i++) - { - thread_invoke(i, tid, f); - } - }); - work += grainsize; - ++tid; - return result; - }); - assert(work >= n); - } -} - -template -void simple_par_for(std::size_t n, std::size_t min_grain, F f) -{ - const auto threadsize = std::min(std::thread::hardware_concurrency(), - n / std::max(1, min_grain)); - simple_par_for_impl(n, threadsize, f); -} - -template -void simple_par_for(std::size_t n, F f) -{ - const int min_grain = 8; - simple_par_for(n, min_grain, f); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/simplify_algebra.hpp b/docker/rocm/migraphx/include/migraphx/simplify_algebra.hpp deleted file mode 100644 index 051e9e0d6..000000000 --- a/docker/rocm/migraphx/include/migraphx/simplify_algebra.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_ALGEBRA_HPP -#define MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_ALGEBRA_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Simplify many algebraic instructions to more efficient versions. - */ -struct MIGRAPHX_EXPORT simplify_algebra -{ - std::string name() const { return "simplify_algebra"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/simplify_dyn_ops.hpp b/docker/rocm/migraphx/include/migraphx/simplify_dyn_ops.hpp deleted file mode 100644 index 22f768ae2..000000000 --- a/docker/rocm/migraphx/include/migraphx/simplify_dyn_ops.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_DYN_OPS_HPP -#define MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_DYN_OPS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Convert dynamic ops to their static version if possible. - * Should be run after the split_single_dyn_dims pass. - */ -struct MIGRAPHX_EXPORT simplify_dyn_ops -{ - std::string name() const { return "simplify_dyn_ops"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/simplify_qdq.hpp b/docker/rocm/migraphx/include/migraphx/simplify_qdq.hpp deleted file mode 100644 index 7be4efc7c..000000000 --- a/docker/rocm/migraphx/include/migraphx/simplify_qdq.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_QDQ_HPP -#define MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_QDQ_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Inserts quantized operators in place of dq->quantizable_op->q - * then removes remaining fake quantization (q->dq pairs) - */ -struct MIGRAPHX_EXPORT simplify_qdq -{ - std::string name() const { return "simplify_qdq"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/simplify_reshapes.hpp b/docker/rocm/migraphx/include/migraphx/simplify_reshapes.hpp deleted file mode 100644 index 8a0901ea5..000000000 --- a/docker/rocm/migraphx/include/migraphx/simplify_reshapes.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_RESHAPES_HPP -#define MIGRAPHX_GUARD_RTGLIB_SIMPLIFY_RESHAPES_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -/** - * Eliminate redundant reshapes. - */ -struct MIGRAPHX_EXPORT simplify_reshapes -{ - size_t depth = 4; - std::string name() const { return "simplify_reshapes"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/source_location.hpp b/docker/rocm/migraphx/include/migraphx/source_location.hpp deleted file mode 100644 index b3952ff8d..000000000 --- a/docker/rocm/migraphx/include/migraphx/source_location.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP - -#include -#include - -#if defined(CPPCHECK) -#define MIGRAPHX_HAS_SOURCE_LOCATION 1 -#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 1 -#elif defined(__has_include) -#if __has_include() && __cplusplus >= 202003L -#define MIGRAPHX_HAS_SOURCE_LOCATION 1 -#else -#define MIGRAPHX_HAS_SOURCE_LOCATION 0 -#endif -#if __has_include() && __cplusplus >= 201103L -#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 1 -#else -#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 0 -#endif -#else -#define MIGRAPHX_HAS_SOURCE_LOCATION 0 -#define MIGRAPHX_HAS_SOURCE_LOCATION_TS 0 -#endif - -#if MIGRAPHX_HAS_SOURCE_LOCATION -#include -#elif MIGRAPHX_HAS_SOURCE_LOCATION_TS -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -#if MIGRAPHX_HAS_SOURCE_LOCATION -using source_location = std::source_location; -#elif MIGRAPHX_HAS_SOURCE_LOCATION_TS -using source_location = std::experimental::source_location; -#else -struct source_location -{ - static constexpr source_location current() noexcept { return source_location{}; } - constexpr std::uint_least32_t line() const noexcept { return 0; } - constexpr std::uint_least32_t column() const noexcept { return 0; } - constexpr const char* file_name() const noexcept { return ""; } - constexpr const char* function_name() const noexcept { return ""; } -}; -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_SOURCE_LOCATION_HPP diff --git a/docker/rocm/migraphx/include/migraphx/split_reduce.hpp b/docker/rocm/migraphx/include/migraphx/split_reduce.hpp deleted file mode 100644 index 687f363b0..000000000 --- a/docker/rocm/migraphx/include/migraphx/split_reduce.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_SPLIT_REDUCE_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_SPLIT_REDUCE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -/// For large reductions that are larger than the split_size, this pass will -/// split the fused_reduce operators so that the reduction will happen across -/// multiple compute units gaining better occupancy for targets with many -/// compute units. Since the reduction is split across compute units, any -/// elementwise operators will be split into separate operators as well due to -/// needing global synchronization. -struct MIGRAPHX_EXPORT split_reduce -{ - std::size_t split_size = 8192; - std::string name() const { return "split_reduce"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_SPLIT_REDUCE_HPP diff --git a/docker/rocm/migraphx/include/migraphx/split_single_dyn_dim.hpp b/docker/rocm/migraphx/include/migraphx/split_single_dyn_dim.hpp deleted file mode 100644 index e92d649a3..000000000 --- a/docker/rocm/migraphx/include/migraphx/split_single_dyn_dim.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_SPLIT_SINGLE_DYN_DIM_HPP -#define MIGRAPHX_GUARD_RTGLIB_SPLIT_SINGLE_DYN_DIM_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Split dynamic dimension over submodules if exactly one dimension in the parameter list is - * dynamic. - */ -struct MIGRAPHX_EXPORT split_single_dyn_dim -{ - std::string name() const { return "split_single_dyn_dim"; } - void apply(module_pass_manager&) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/sqlite.hpp b/docker/rocm/migraphx/include/migraphx/sqlite.hpp deleted file mode 100644 index 97f3c0405..000000000 --- a/docker/rocm/migraphx/include/migraphx/sqlite.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_SQLITE_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_SQLITE_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct sqlite_impl; - -struct MIGRAPHX_EXPORT sqlite -{ - sqlite() = default; - static sqlite read(const fs::path& p); - static sqlite write(const fs::path& p); - std::vector> execute(const std::string& s); - - private: - std::shared_ptr impl; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_SQLITE_HPP diff --git a/docker/rocm/migraphx/include/migraphx/stream_model.hpp b/docker/rocm/migraphx/include/migraphx/stream_model.hpp deleted file mode 100644 index d95ccda0c..000000000 --- a/docker/rocm/migraphx/include/migraphx/stream_model.hpp +++ /dev/null @@ -1,377 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_STREAM_MODEL_HPP -#define MIGRAPHX_GUARD_STREAM_MODEL_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#ifdef DOXYGEN - -/// An interface for target-dependent model for the scheduler -struct stream_model -{ - /// Get the number of streams used in the program - std::size_t get_nstream() const; - /// Get stream for instruction - std::size_t get_stream(instruction_ref ins) const; - /// Get unique event id for instruction - std::size_t get_event_id(instruction_ref ins) const; - /// Returns true if instruction has a stream assignment - bool has_stream(instruction_ref ins) const; - /// Returns true if the instruction records the event - bool is_record(instruction_ref ins) const; - /// Returns true if the instruction wait on the event - bool is_wait(instruction_ref ins) const; -}; - -#else - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT stream_model -{ - // - std::size_t get_nstream() const; - // - std::size_t get_stream(instruction_ref ins) const; - // - std::size_t get_event_id(instruction_ref ins) const; - // - bool has_stream(instruction_ref ins) const; - // - bool is_record(instruction_ref ins) const; - // - bool is_wait(instruction_ref ins) const; -}; - -#else - -struct stream_model -{ - private: - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().get_nstream(), - std::declval().get_stream( - std::declval()), - std::declval().get_event_id( - std::declval()), - std::declval().has_stream( - std::declval()), - std::declval().is_record( - std::declval()), - std::declval().is_wait(std::declval()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - stream_model() = default; - - template < - typename PrivateDetailTypeErasedT, - typename = private_te_constraints, - typename = typename std::enable_if< - not std::is_same, stream_model>{}>::type> - stream_model(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template < - typename PrivateDetailTypeErasedT, - typename = private_te_constraints, - typename = typename std::enable_if< - not std::is_same, stream_model>{}>::type> - stream_model& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - stream_model rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::size_t get_nstream() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_nstream(); - } - - std::size_t get_stream(instruction_ref ins) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_stream(ins); - } - - std::size_t get_event_id(instruction_ref ins) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_event_id(ins); - } - - bool has_stream(instruction_ref ins) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().has_stream(ins); - } - - bool is_record(instruction_ref ins) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().is_record(ins); - } - - bool is_wait(instruction_ref ins) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().is_wait(ins); - } - - friend bool is_shared(const stream_model& private_detail_x, - const stream_model& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::size_t get_nstream() const = 0; - virtual std::size_t get_stream(instruction_ref ins) const = 0; - virtual std::size_t get_event_id(instruction_ref ins) const = 0; - virtual bool has_stream(instruction_ref ins) const = 0; - virtual bool is_record(instruction_ref ins) const = 0; - virtual bool is_wait(instruction_ref ins) const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::size_t get_nstream() const override { return private_detail_te_value.get_nstream(); } - - std::size_t get_stream(instruction_ref ins) const override - { - - return private_detail_te_value.get_stream(ins); - } - - std::size_t get_event_id(instruction_ref ins) const override - { - - return private_detail_te_value.get_event_id(ins); - } - - bool has_stream(instruction_ref ins) const override - { - - return private_detail_te_value.has_stream(ins); - } - - bool is_record(instruction_ref ins) const override - { - - return private_detail_te_value.is_record(ins); - } - - bool is_wait(instruction_ref ins) const override - { - - return private_detail_te_value.is_wait(ins); - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const stream_model* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(stream_model* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(stream_model& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const stream_model& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -#endif - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/streamutils.hpp b/docker/rocm/migraphx/include/migraphx/streamutils.hpp deleted file mode 100644 index 6b9fa2d56..000000000 --- a/docker/rocm/migraphx/include/migraphx/streamutils.hpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_STREAMUTILS_HPP -#define MIGRAPHX_GUARD_STREAMUTILS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct stream_range_container -{ - const T* r; - stream_range_container(const T& x) : r(&x) {} - - friend std::ostream& operator<<(std::ostream& os, const stream_range_container& sr) - { - assert(sr.r != nullptr); - if(not sr.r->empty()) - { - os << sr.r->front(); - std::for_each( - std::next(sr.r->begin()), sr.r->end(), [&](auto&& x) { os << ", " << x; }); - } - return os; - } -}; - -template -inline stream_range_container stream_range(const Range& r) -{ - return {r}; -} - -namespace detail { - -template -auto stream_write_value_impl(rank<1>, std::ostream& os, const T& x) -> decltype(os << x, void()) -{ - os << x; -} - -template -auto stream_write_value_impl(rank<1>, std::ostream& os, const optional& x) -{ - if(x.has_value()) - { - os << *x; - } - else - { - os << "nullopt"; - } -} - -template -void stream_write_value_impl(rank<1>, std::ostream& os, const std::vector& r) -{ - os << "{"; - os << stream_range(r); - os << "}"; -} - -template -auto stream_write_value_impl(rank<0>, std::ostream& os, const Range& r) - -> decltype(r.begin(), r.end(), void()) -{ - os << "{"; - os << stream_range(r); - os << "}"; -} - -template {})> -void stream_write_value_impl(rank<0>, std::ostream& os, const T& x) -{ - char delim = '{'; - reflect_each(x, [&](auto&& y, auto name) { - os << delim; - os << name << "="; - stream_write_value_impl(rank<2>{}, os, y); - delim = ','; - }); - if(delim == ',') - os << "}"; -} - -} // namespace detail - -template -void stream_write_value(std::ostream& os, const T& x) -{ - detail::stream_write_value_impl(rank<1>{}, os, x); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/stringutils.hpp b/docker/rocm/migraphx/include/migraphx/stringutils.hpp deleted file mode 100644 index 47c13fd35..000000000 --- a/docker/rocm/migraphx/include/migraphx/stringutils.hpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_STRINGUTILS_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_STRINGUTILS_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#define MIGRAPHX_STRINGIZE_1(...) #__VA_ARGS__ -#define MIGRAPHX_STRINGIZE(...) MIGRAPHX_STRINGIZE_1(__VA_ARGS__) - -template -auto with_char(F f) -{ - return [=](unsigned char c) -> bool { return f(c); }; -} - -inline void -replace_string_inplace(std::string& subject, const std::string& search, const std::string& replace) -{ - size_t pos = 0; - while((pos = subject.find(search, pos)) != std::string::npos) - { - subject.replace(pos, search.length(), replace); - pos += replace.length(); - } -} - -inline std::string -replace_string(std::string subject, const std::string& search, const std::string& replace) -{ - replace_string_inplace(subject, search, replace); - return subject; -} - -inline bool ends_with(const std::string& value, const std::string& suffix) -{ - if(suffix.size() > value.size()) - return false; - else - return std::equal(suffix.rbegin(), suffix.rend(), value.rbegin()); -} - -template -inline std::string join_strings(Strings strings, const std::string& delim) -{ - auto it = strings.begin(); - if(it == strings.end()) - return ""; - - auto nit = std::next(it); - return std::accumulate(nit, strings.end(), *it, [&](std::string x, std::string y) { - return std::move(x) + delim + std::move(y); - }); -} - -inline std::vector split_string(const std::string& s, char delim) -{ - std::vector elems; - std::stringstream ss(s + delim); - std::string item; - while(std::getline(ss, item, delim)) - { - elems.push_back(item); - } - return elems; -} - -template -std::string trim(const std::string& s, F f) -{ - auto start = std::find_if_not(s.begin(), s.end(), f); - auto last = std::find_if_not(s.rbegin(), std::string::const_reverse_iterator(start), f).base(); - return {start, last}; -} - -inline std::string trim(const std::string& s) -{ - return trim(s, [](unsigned char c) { return std::isspace(c); }); -} - -template -inline std::string transform_string(std::string s, F f) -{ - std::transform(s.begin(), s.end(), s.begin(), f); - return s; -} - -inline std::string to_upper(std::string s) { return transform_string(std::move(s), ::toupper); } - -inline std::string to_lower(std::string s) { return transform_string(std::move(s), ::tolower); } - -inline bool starts_with(const std::string& value, const std::string& prefix) -{ - if(prefix.size() > value.size()) - return false; - else - return std::equal(prefix.begin(), prefix.end(), value.begin()); -} - -inline std::string remove_prefix(std::string s, const std::string& prefix) -{ - if(starts_with(s, prefix)) - return s.substr(prefix.length()); - else - return s; -} - -template -inline std::string -interpolate_string(const std::string& input, F f, std::string start = "${", std::string end = "}") -{ - std::string result = ""; - result.reserve(input.size()); - auto it = input.begin(); - while(it != input.end()) - { - auto next_start = std::search(it, input.end(), start.begin(), start.end()); - auto next_end = std::search(next_start, input.end(), end.begin(), end.end()); - result.append(it, next_start); - if(next_start == input.end()) - break; - if(next_end == input.end()) - { - throw std::runtime_error("Unbalanced brackets"); - } - auto r = f(next_start + start.size(), next_end); - result.append(r.begin(), r.end()); - it = next_end + end.size(); - } - return result; -} -inline std::string interpolate_string(const std::string& input, - const std::unordered_map& vars, - std::string start = "${", - std::string end = "}") -{ - return interpolate_string( - input, - [&](auto start_it, auto last_it) { - auto key = trim({start_it, last_it}); - auto it = vars.find(key); - if(it == vars.end()) - throw std::runtime_error("Unknown key: " + key); - return it->second; - }, - std::move(start), - std::move(end)); -} - -inline std::string to_c_id(const std::string& name, char rep = '_') -{ - std::string id = transform_string(name, [&](auto c) { - if(with_char(::isalnum)(c) or c == '_') - return c; - return rep; - }); - while(id.find("__") != std::string::npos) - replace_string_inplace(id, "__", "_"); - return id; -} - -inline std::string quote_string(const std::string& str) { return "\"" + str + "\""; } - -template -inline std::string to_string_range(Iterator start, Iterator last, const char* delim = ", ") -{ - std::stringstream ss; - if(start != last) - { - ss << as_number(*start); - std::for_each(std::next(start), last, [&](auto&& x) { ss << delim << as_number(x); }); - } - return ss.str(); -} - -template -inline std::string to_string_range(const Range& r, const char* delim = ", ") -{ - return to_string_range(r.begin(), r.end(), delim); -} - -template -inline std::string to_string_range(const std::initializer_list& r, const char* delim = ", ") -{ - return to_string_range(r.begin(), r.end(), delim); -} - -template -inline auto to_string(const T& x) - -> decltype((std::declval() << x), std::string{}) -{ - std::stringstream ss; - ss << x; - return ss.str(); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/support_metric.hpp b/docker/rocm/migraphx/include/migraphx/support_metric.hpp deleted file mode 100644 index c470287e7..000000000 --- a/docker/rocm/migraphx/include/migraphx/support_metric.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_SUPPORT_METRIC_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_SUPPORT_METRIC_HPP - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -enum class support_metric -{ - latency, - throughput -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_SUPPORT_METRIC_HPP diff --git a/docker/rocm/migraphx/include/migraphx/supported_segments.hpp b/docker/rocm/migraphx/include/migraphx/supported_segments.hpp deleted file mode 100644 index d1026b33a..000000000 --- a/docker/rocm/migraphx/include/migraphx/supported_segments.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_SUPPORTED_SEGMENTS_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_SUPPORTED_SEGMENTS_HPP - -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct supported_segment -{ - std::unordered_set instructions; - float metric; -}; - -using supported_segments = std::vector; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_SUPPORTED_SEGMENTS_HPP diff --git a/docker/rocm/migraphx/include/migraphx/target.hpp b/docker/rocm/migraphx/include/migraphx/target.hpp deleted file mode 100644 index bf105286a..000000000 --- a/docker/rocm/migraphx/include/migraphx/target.hpp +++ /dev/null @@ -1,525 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_TARGET_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_TARGET_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct value; - -#ifdef DOXYGEN - -/// An interface for a compilation target -struct target -{ - /// A unique name used to identify the target - std::string name() const; - /** - * @brief The transformation pass to be run during compilation. - * - * @param ctx This is the target-dependent context that is created by `get_context` - * @param options Compiling options passed in by the user - * @return The passes to be ran - */ - std::vector get_passes(context& ctx, const compile_options& options) const; - /** - * @brief Construct a context for the target. - * @return The context to be used during compilation and execution. - */ - context get_context() const; - /** - * @brief Get the ranges of instructions that are supported on a target - * @param module Module to check for supported instructions - * @param metric Used to define how the quality of the support should be measured - * @return the supported segments of the graph - */ - supported_segments target_is_supported(T&, const_module_ref mod, support_metric metric) const; - /** - * @brief copy an argument to the current target. - * - * @param arg Input argument to be copied to the target - * @return Argument in the target. - */ - argument copy_to(const argument& arg) const; - /** - * @brief copy an argument from the current target. - * - * @param arg Input argument to be copied from the target - * @return Argument in the host. - */ - argument copy_from(const argument& arg) const; - /** - * @brief Allocate an argument based on the input shape - * - * @param s Shape of the argument to be allocated in the target - * @return Allocated argument in the target. - */ - argument allocate(const shape& s) const; -}; - -#else - -template -argument target_allocate(T& x, const shape&) -{ - std::string name = x.name(); - MIGRAPHX_THROW("Not computable: " + name); -} - -template -argument copy_to_target(T&, const argument& arg) -{ - return arg; -} - -template -argument copy_from_target(T&, const argument& arg) -{ - return arg; -} - -template -supported_segments target_find_supported(T&, const_module_ref, support_metric) -{ - return {}; -} - -#ifdef TYPE_ERASED_DECLARATION - -// Type-erased interface for: -struct MIGRAPHX_EXPORT target -{ - // - std::string name() const; - // - std::vector get_passes(context& ctx, const compile_options& options) const; - // - context get_context() const; - // (optional) - supported_segments find_supported(const_module_ref mod, support_metric m) const; - // (optional) - argument copy_to(const argument& input) const; - // (optional) - argument copy_from(const argument& input) const; - // (optional) - argument allocate(const shape& s) const; -}; - -#else - -struct target -{ - private: - template - static auto private_detail_te_default_find_supported(char, - T&& private_detail_te_self, - const_module_ref mod, - support_metric m) - -> decltype(private_detail_te_self.find_supported(mod, m)) - { - return private_detail_te_self.find_supported(mod, m); - } - - template - static supported_segments private_detail_te_default_find_supported(float, - T&& private_detail_te_self, - const_module_ref mod, - support_metric m) - { - return target_find_supported(private_detail_te_self, mod, m); - } - - template - static auto - private_detail_te_default_copy_to(char, T&& private_detail_te_self, const argument& input) - -> decltype(private_detail_te_self.copy_to(input)) - { - return private_detail_te_self.copy_to(input); - } - - template - static argument - private_detail_te_default_copy_to(float, T&& private_detail_te_self, const argument& input) - { - return copy_to_target(private_detail_te_self, input); - } - - template - static auto - private_detail_te_default_copy_from(char, T&& private_detail_te_self, const argument& input) - -> decltype(private_detail_te_self.copy_from(input)) - { - return private_detail_te_self.copy_from(input); - } - - template - static argument - private_detail_te_default_copy_from(float, T&& private_detail_te_self, const argument& input) - { - return copy_from_target(private_detail_te_self, input); - } - - template - static auto private_detail_te_default_allocate(char, T&& private_detail_te_self, const shape& s) - -> decltype(private_detail_te_self.allocate(s)) - { - return private_detail_te_self.allocate(s); - } - - template - static argument - private_detail_te_default_allocate(float, T&& private_detail_te_self, const shape& s) - { - return target_allocate(private_detail_te_self, s); - } - - template - struct private_te_unwrap_reference - { - using type = PrivateDetailTypeErasedT; - }; - template - struct private_te_unwrap_reference> - { - using type = PrivateDetailTypeErasedT; - }; - template - using private_te_pure = typename std::remove_cv< - typename std::remove_reference::type>::type; - - template - using private_te_constraints_impl = - decltype(std::declval().name(), - std::declval().get_passes( - std::declval(), std::declval()), - std::declval().get_context(), - private_detail_te_default_find_supported(char(0), - std::declval(), - std::declval(), - std::declval()), - private_detail_te_default_copy_to(char(0), - std::declval(), - std::declval()), - private_detail_te_default_copy_from(char(0), - std::declval(), - std::declval()), - private_detail_te_default_allocate(char(0), - std::declval(), - std::declval()), - void()); - - template - using private_te_constraints = private_te_constraints_impl< - typename private_te_unwrap_reference>::type>; - - public: - // Constructors - target() = default; - - template , - typename = typename std::enable_if< - not std::is_same, target>{}>::type> - target(PrivateDetailTypeErasedT&& value) - : private_detail_te_handle_mem_var( - std::make_shared< - private_detail_te_handle_type>>( - std::forward(value))) - { - } - - // Assignment - template , - typename = typename std::enable_if< - not std::is_same, target>{}>::type> - target& operator=(PrivateDetailTypeErasedT&& value) - { - using std::swap; - auto* derived = this->any_cast>(); - if(derived and private_detail_te_handle_mem_var.use_count() == 1) - { - *derived = std::forward(value); - } - else - { - target rhs(value); - swap(private_detail_te_handle_mem_var, rhs.private_detail_te_handle_mem_var); - } - return *this; - } - - // Cast - template - PrivateDetailTypeErasedT* any_cast() - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - template - const typename std::remove_cv::type* any_cast() const - { - return this->type_id() == typeid(PrivateDetailTypeErasedT) - ? std::addressof(static_cast::type>&>( - private_detail_te_get_handle()) - .private_detail_te_value) - : nullptr; - } - - const std::type_info& type_id() const - { - if(private_detail_te_handle_empty()) - return typeid(std::nullptr_t); - else - return private_detail_te_get_handle().type(); - } - - std::string name() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().name(); - } - - std::vector get_passes(context& ctx, const compile_options& options) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_passes(ctx, options); - } - - context get_context() const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().get_context(); - } - - supported_segments find_supported(const_module_ref mod, support_metric m) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().find_supported(mod, m); - } - - argument copy_to(const argument& input) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().copy_to(input); - } - - argument copy_from(const argument& input) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().copy_from(input); - } - - argument allocate(const shape& s) const - { - assert((*this).private_detail_te_handle_mem_var); - return (*this).private_detail_te_get_handle().allocate(s); - } - - friend bool is_shared(const target& private_detail_x, const target& private_detail_y) - { - return private_detail_x.private_detail_te_handle_mem_var == - private_detail_y.private_detail_te_handle_mem_var; - } - - private: - struct private_detail_te_handle_base_type - { - virtual ~private_detail_te_handle_base_type() {} - virtual std::shared_ptr clone() const = 0; - virtual const std::type_info& type() const = 0; - - virtual std::string name() const = 0; - virtual std::vector get_passes(context& ctx, - const compile_options& options) const = 0; - virtual context get_context() const = 0; - virtual supported_segments find_supported(const_module_ref mod, support_metric m) const = 0; - virtual argument copy_to(const argument& input) const = 0; - virtual argument copy_from(const argument& input) const = 0; - virtual argument allocate(const shape& s) const = 0; - }; - - template - struct private_detail_te_handle_type : private_detail_te_handle_base_type - { - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value>::type* = - nullptr) - : private_detail_te_value(value) - { - } - - template - private_detail_te_handle_type( - PrivateDetailTypeErasedT value, - typename std::enable_if::value, - int>::type* = nullptr) noexcept - : private_detail_te_value(std::move(value)) - { - } - - std::shared_ptr clone() const override - { - return std::make_shared(private_detail_te_value); - } - - const std::type_info& type() const override { return typeid(private_detail_te_value); } - - std::string name() const override { return private_detail_te_value.name(); } - - std::vector get_passes(context& ctx, const compile_options& options) const override - { - - return private_detail_te_value.get_passes(ctx, options); - } - - context get_context() const override { return private_detail_te_value.get_context(); } - - supported_segments find_supported(const_module_ref mod, support_metric m) const override - { - - return private_detail_te_default_find_supported( - char(0), private_detail_te_value, mod, m); - } - - argument copy_to(const argument& input) const override - { - - return private_detail_te_default_copy_to(char(0), private_detail_te_value, input); - } - - argument copy_from(const argument& input) const override - { - - return private_detail_te_default_copy_from(char(0), private_detail_te_value, input); - } - - argument allocate(const shape& s) const override - { - - return private_detail_te_default_allocate(char(0), private_detail_te_value, s); - } - - PrivateDetailTypeErasedT private_detail_te_value; - }; - - template - struct private_detail_te_handle_type> - : private_detail_te_handle_type - { - private_detail_te_handle_type(std::reference_wrapper ref) - : private_detail_te_handle_type(ref.get()) - { - } - }; - - bool private_detail_te_handle_empty() const - { - return private_detail_te_handle_mem_var == nullptr; - } - - const private_detail_te_handle_base_type& private_detail_te_get_handle() const - { - assert(private_detail_te_handle_mem_var != nullptr); - return *private_detail_te_handle_mem_var; - } - - private_detail_te_handle_base_type& private_detail_te_get_handle() - { - assert(private_detail_te_handle_mem_var != nullptr); - if(private_detail_te_handle_mem_var.use_count() > 1) - private_detail_te_handle_mem_var = private_detail_te_handle_mem_var->clone(); - return *private_detail_te_handle_mem_var; - } - - std::shared_ptr private_detail_te_handle_mem_var; -}; - -template -inline const ValueType* any_cast(const target* x) -{ - return x->any_cast(); -} - -template -inline ValueType* any_cast(target* x) -{ - return x->any_cast(); -} - -template -inline ValueType& any_cast(target& x) -{ - auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} - -template -inline const ValueType& any_cast(const target& x) -{ - const auto* y = x.any_cast::type>(); - if(y == nullptr) - throw std::bad_cast(); - return *y; -} -#endif - -#endif - -void migraphx_to_value(value& v, const target& t); -void migraphx_from_value(const value& v, target& t); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/target_assignments.hpp b/docker/rocm/migraphx/include/migraphx/target_assignments.hpp deleted file mode 100644 index 8064d4d03..000000000 --- a/docker/rocm/migraphx/include/migraphx/target_assignments.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHX_ASSIGNMENT_HPP -#define MIGRAPHX_GUARD_MIGRAPHX_ASSIGNMENT_HPP - -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct target_assignments -{ - using iterator = std::unordered_map::const_iterator; - using value_type = std::pair; - - auto size() const { return assignments.size(); } - auto& at(instruction_ref ins) const { return assignments.at(ins); } - - auto insert(iterator it, const std::pair& assignment) - { - return assignments.insert(it, assignment); - } - auto find(instruction_ref ins) const { return assignments.find(ins); } - - auto begin() const { return assignments.begin(); } - auto end() const { return assignments.end(); } - - private: - std::unordered_map assignments; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_MIGRAPHX_ASSIGNMENT_HPP diff --git a/docker/rocm/migraphx/include/migraphx/tensor_view.hpp b/docker/rocm/migraphx/include/migraphx/tensor_view.hpp deleted file mode 100644 index a8a3579d4..000000000 --- a/docker/rocm/migraphx/include/migraphx/tensor_view.hpp +++ /dev/null @@ -1,212 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_TENSOR_VIEW_HPP -#define MIGRAPHX_GUARD_TENSOR_VIEW_HPP - -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -struct tensor_view_iterator_read -{ - T* view; - auto& operator()(std::size_t n) const - { - assert(view != nullptr); - return (*view)[n]; - } -}; - -template -struct tensor_view -{ - using value_type = T; - using iterator = basic_iota_iterator>, std::size_t>; - using const_iterator = - basic_iota_iterator>, std::size_t>; - tensor_view() : m_data(nullptr) {} - tensor_view(shape s, T* d) : m_data(d), m_shape(std::move(s)) {} - - const shape& get_shape() const { return this->m_shape; } - - bool empty() const { return m_data == nullptr or m_shape.lens().empty(); } - - std::size_t size() const { return m_shape.elements(); } - - T* data() { return this->m_data; } - - const T* data() const { return this->m_data; } - - template {}...)> - const T& operator()(Ts... xs) const - { - assert(std::vector{static_cast(xs)...} < m_shape.lens()); - assert(m_shape.index({static_cast(xs)...}) < m_shape.bytes() / sizeof(T)); - return m_data[m_shape.index({static_cast(xs)...})]; - } - - template {}...)> - T& operator()(Ts... xs) - { - assert(std::vector{static_cast(xs)...} < m_shape.lens()); - assert(m_shape.index({static_cast(xs)...}) < m_shape.bytes() / sizeof(T)); - return m_data[m_shape.index({static_cast(xs)...})]; - } - - template {})> - const T& operator()(Iterator start, Iterator last) const - { - assert(std::distance(start, last) > 0); - assert(std::all_of(start, last, [](auto x) { return x >= 0; })); - return m_data[m_shape.index(start, last)]; - } - - template {})> - T& operator()(Iterator start, Iterator last) - { - assert(std::distance(start, last) > 0); - assert(std::all_of(start, last, [](auto x) { return x >= 0; })); - return m_data[m_shape.index(start, last)]; - } - - T& operator[](std::size_t i) - { - assert(not this->empty() and i < this->size()); - return m_data[m_shape.index(i)]; - } - - const T& operator[](std::size_t i) const - { - assert(not this->empty() and i < this->size()); - return m_data[m_shape.index(i)]; - } - - template - auto operator[](const Range& r) -> decltype((*this)(r.begin(), r.end())) - { - return (*this)(r.begin(), r.end()); - } - - template - auto operator[](const Range& r) const -> decltype((*this)(r.begin(), r.end())) - { - return (*this)(r.begin(), r.end()); - } - - T& front() - { - assert(not this->empty()); - return m_data[0]; - } - - const T& front() const - { - assert(not this->empty()); - return m_data[0]; - } - - T& back() - { - assert(not this->empty()); - return m_data[m_shape.index(this->size() - 1)]; - } - - const T& back() const - { - assert(not this->empty()); - return m_data[m_shape.index(this->size() - 1)]; - } - - iterator begin() { return {0, {this}}; } - - iterator end() { return {this->size(), {this}}; } - - const_iterator begin() const { return {0, {this}}; } - - const_iterator end() const { return {this->size(), {this}}; } - - template - std::vector to_vector() const - { - return std::vector(this->begin(), this->end()); - } - - friend std::ostream& operator<<(std::ostream& os, const tensor_view& x) - { - if(not x.empty()) - { - os << as_number(x.front()); - for(std::size_t i = 1; i < x.m_shape.elements(); i++) - { - os << ", " << as_number(x.m_data[x.m_shape.index(i)]); - } - } - return os; - } - - private: - T* m_data; - shape m_shape; -}; - -template -bool operator==(const tensor_view& x, const tensor_view& y) -{ - if(x.get_shape() == y.get_shape()) - { - for(std::size_t i = 0; i < x.get_shape().elements(); i++) - { - if(not float_equal(x[i], y[i])) - return false; - } - return true; - } - return false; -} - -template -bool operator!=(const tensor_view& x, const tensor_view& y) -{ - return not(x == y); -} - -template -tensor_view make_view(const shape& s, T* data) -{ - return {s, data}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/tf.hpp b/docker/rocm/migraphx/include/migraphx/tf.hpp deleted file mode 100644 index 2f8fa536f..000000000 --- a/docker/rocm/migraphx/include/migraphx/tf.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_TF_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_TF_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// struct to pass in tf options to parser -struct tf_options -{ - bool is_nhwc = false; - unsigned int batch_size = 1; - /// Explicitly specify the dims of an input - std::unordered_map> map_input_dims = {}; - std::vector output_node_names = {}; -}; - -/// Create a program from a tf pb file (default is nhwc format) -MIGRAPHX_TF_EXPORT program parse_tf(const std::string& name, - const tf_options& options = tf_options{}); - -/// Create a program from an tf buffer -MIGRAPHX_TF_EXPORT program parse_tf_buffer(const std::string& buffer, - const tf_options& options = tf_options{}); - -/// Create a program from tf buffer -MIGRAPHX_TF_EXPORT program parse_tf_buffer(const void* data, - std::size_t size, - const tf_options& options = tf_options{}); - -MIGRAPHX_TF_EXPORT std::vector get_tf_operators(); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/time.hpp b/docker/rocm/migraphx/include/migraphx/time.hpp deleted file mode 100644 index c111006ce..000000000 --- a/docker/rocm/migraphx/include/migraphx/time.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_TIME_HPP -#define MIGRAPHX_GUARD_RTGLIB_TIME_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct timer -{ - std::chrono::time_point start = std::chrono::steady_clock::now(); - template - auto record() const - { - auto finish = std::chrono::steady_clock::now(); - return std::chrono::duration_cast(finish - start).count(); - } -}; - -template -auto time(F f) -{ - timer t{}; - f(); - return t.record(); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/tmp_dir.hpp b/docker/rocm/migraphx/include/migraphx/tmp_dir.hpp deleted file mode 100644 index f047a615c..000000000 --- a/docker/rocm/migraphx/include/migraphx/tmp_dir.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_TMP_DIR_HPP -#define MIGRAPHX_GUARD_RTGLIB_TMP_DIR_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct MIGRAPHX_EXPORT tmp_dir -{ - fs::path path; - tmp_dir(std::string_view prefix = ""); - tmp_dir(tmp_dir&&) = default; - - void execute(std::string_view cmd, const std::vector& args = {}) const; - void execute(const fs::path& cmd, const std::vector& args = {}) const - { - execute(std::string_view{cmd.string()}, args); - } - - tmp_dir(tmp_dir const&) = delete; - tmp_dir& operator=(tmp_dir const&) = delete; - - ~tmp_dir(); -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/tracer.hpp b/docker/rocm/migraphx/include/migraphx/tracer.hpp deleted file mode 100644 index 9a938a0f2..000000000 --- a/docker/rocm/migraphx/include/migraphx/tracer.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_TRACER_HPP -#define MIGRAPHX_GUARD_RTGLIB_TRACER_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct tracer -{ - tracer() {} - - tracer(std::ostream& s) : os(&s) {} - - bool enabled() const { return os != nullptr; } - - template - void operator()(const Ts&... xs) const - { - if(os != nullptr) - { - swallow{*os << xs...}; - *os << std::endl; - } - } - - private: - std::ostream* os = nullptr; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/truncate_float.hpp b/docker/rocm/migraphx/include/migraphx/truncate_float.hpp deleted file mode 100644 index 426a445c0..000000000 --- a/docker/rocm/migraphx/include/migraphx/truncate_float.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_TRUNCATE_FLOAT_HPP -#define MIGRAPHX_GUARD_RTGLIB_TRUNCATE_FLOAT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct program; -struct module; - -/** - * quantize a program to fp - */ -struct MIGRAPHX_EXPORT truncate_float_pass -{ - std::vector ins_names = {"all"}; - shape::type_t float_type = shape::float_type; - std::string name() const { return "truncate_float"; } - void apply(module& m) const; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/tune_axis.hpp b/docker/rocm/migraphx/include/migraphx/tune_axis.hpp deleted file mode 100644 index a8d7acc06..000000000 --- a/docker/rocm/migraphx/include/migraphx/tune_axis.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_OPERATORS_TUNE_AXIS_HPP -#define MIGRAPHX_GUARD_OPERATORS_TUNE_AXIS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -inline int tune_axis(int n_dim, int axis, const std::string& name = "OPERATOR") -{ - if(axis < 0) - axis += n_dim; - - if(axis < 0 or axis >= n_dim) - MIGRAPHX_THROW(to_upper(name) + ": axis is out of range."); - - return axis; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/type_name.hpp b/docker/rocm/migraphx/include/migraphx/type_name.hpp deleted file mode 100644 index 50396e1a4..000000000 --- a/docker/rocm/migraphx/include/migraphx/type_name.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_TYPE_NAME_HPP -#define MIGRAPHX_GUARD_RTGLIB_TYPE_NAME_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -std::string compute_type_name() -{ - const char parameter_name[] = "PrivateMigraphTypeNameProbe ="; // NOLINT - - std::string name = __PRETTY_FUNCTION__; - - auto begin = name.find(parameter_name) + sizeof(parameter_name); -#if(defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7) - auto length = name.find_last_of(",") - begin; -#else - auto length = name.find_first_of("];", begin) - begin; -#endif - return name.substr(begin, length); -} - -template -const std::string& get_type_name() -{ - static const std::string name = compute_type_name(); - return name; -} - -template -const std::string& get_type_name(const T&) -{ - return migraphx::get_type_name(); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/type_traits.hpp b/docker/rocm/migraphx/include/migraphx/type_traits.hpp deleted file mode 100644 index ea42f83af..000000000 --- a/docker/rocm/migraphx/include/migraphx/type_traits.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* -* The MIT License (MIT) -* -* Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in -* all copies or substantial portions of the Software. - -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -* THE SOFTWARE. -*/ - -#ifndef MIGRAPHX_GUARD_RTGLIB_TYPE_TRAITS_HPP -#define MIGRAPHX_GUARD_RTGLIB_TYPE_TRAITS_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -#define MIGRAPHX_DETAIL_DEFINE_TRAIT(trait) \ - template \ - struct trait : std::trait \ - { \ - }; - -#define MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(trait, T) \ - template <> \ - struct trait : std::true_type \ - { \ - }; - -MIGRAPHX_DETAIL_DEFINE_TRAIT(is_floating_point); -MIGRAPHX_DETAIL_DEFINE_TRAIT(is_arithmetic); -MIGRAPHX_DETAIL_DEFINE_TRAIT(is_signed); - -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, half) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_signed, half) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, half) - -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, bf16) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_signed, bf16) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, bf16) - -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, migraphx::fp8::fp8e4m3fnuz) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_signed, migraphx::fp8::fp8e4m3fnuz) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, migraphx::fp8::fp8e4m3fnuz) - -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, migraphx::fp8::fp8e5m2fnuz) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_signed, migraphx::fp8::fp8e5m2fnuz) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, migraphx::fp8::fp8e5m2fnuz) - -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, migraphx::fp8::fp8e4m3fn) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_signed, migraphx::fp8::fp8e4m3fn) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, migraphx::fp8::fp8e4m3fn) - -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_floating_point, migraphx::fp8::fp8e5m2) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_signed, migraphx::fp8::fp8e5m2) -MIGRAPHX_DETAIL_EXTEND_TRAIT_FOR(is_arithmetic, migraphx::fp8::fp8e5m2) - -template -using accumulator_type = - std::conditional_t{}, - double, - std::conditional_t{}, std::int64_t, std::uint64_t>>; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/value.hpp b/docker/rocm/migraphx/include/migraphx/value.hpp deleted file mode 100644 index 5efb80176..000000000 --- a/docker/rocm/migraphx/include/migraphx/value.hpp +++ /dev/null @@ -1,498 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_VALUE_HPP -#define MIGRAPHX_GUARD_RTGLIB_VALUE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct value_base_impl; - -template -struct value_converter -{ - template - static auto apply(const std::string& x) - -> decltype((std::declval() >> std::declval()), To{}) - { - To result; - std::stringstream ss; - ss.str(x); - ss >> result; - if(ss.fail()) - throw std::runtime_error("Failed to parse: " + x); - return result; - } - - template {})> - static To apply(const From& x) - { - return To(x); - } -}; - -template -struct value_converter{})> -{ - template - static auto apply(const From& x) - -> decltype(static_cast(value_converter>::apply(x))) - { - return static_cast(value_converter>::apply(x)); - } -}; - -template <> -struct value_converter -{ - static const std::string& apply(const std::string& x) { return x; } - - template - static auto apply(const From& x) - -> decltype(std::declval() << x, std::string()) - { - std::stringstream ss; - ss << x; - if(ss.fail()) - throw std::runtime_error("Failed to parse"); - return ss.str(); - } -}; - -template -struct value_converter> -{ - template - static auto apply(const std::pair& x) - -> decltype(std::pair(x.first, value_converter::apply(x.second))) - { - return std::pair(x.first, value_converter::apply(x.second)); - } -}; - -template -To try_convert_value(const From& x); - -namespace detail { -template -To try_convert_value_impl(rank<1>, const std::pair& x) -{ - return try_convert_value(x.second); -} - -template -auto try_convert_value_impl(rank<2>, const From& x) -> decltype(value_converter::apply(x)) -{ - return value_converter::apply(x); -} - -template {})> -To try_convert_value_impl(rank<3>, std::nullptr_t) -{ - MIGRAPHX_THROW("Incompatible values: null -> " + get_type_name()); -} - -template -To try_convert_value_impl(rank<0>, const From& x) -{ - MIGRAPHX_THROW("Incompatible values: " + get_type_name(x) + " -> " + get_type_name()); -} -} // namespace detail - -template -To try_convert_value(const From& x) -{ - return detail::try_convert_value_impl(rank<3>{}, x); -} - -struct MIGRAPHX_EXPORT value -{ -// clang-format off -#define MIGRAPHX_VISIT_VALUE_TYPES(m) \ - m(int64, std::int64_t) \ - m(uint64, std::uint64_t) \ - m(float, double) \ - m(string, std::string) \ - m(bool, bool) \ - m(binary, value::binary) - // clang-format on - enum type_t - { -#define MIGRAPHX_VALUE_GENERATE_ENUM_TYPE(vt, cpp_type) vt##_type, - MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_ENUM_TYPE) object_type, - array_type, - null_type -#undef MIGRAPHX_VALUE_GENERATE_ENUM_TYPE - }; - using iterator = value*; - using const_iterator = const value*; - using value_type = value; - using key_type = std::string; - using mapped_type = value; - using reference = value_type&; - using const_reference = const value_type&; - using pointer = value_type*; - using const_pointer = const value_type*; - using array = std::vector; - using object = std::unordered_map; - struct binary : std::vector - { - using base = std::vector; - binary() {} - template ().begin()) == 1)> - explicit binary(const Container& c) : base(c.begin(), c.end()) - { - } - template - binary(T* data, std::size_t s) : base(data, data + s) - { - } - explicit binary(std::size_t s) : base(s) {} - - friend std::ostream& operator<<(std::ostream& os, const binary& obj) - { - os << "{binary_object: " << obj.size() << "}"; - return os; - } - }; - - value() = default; - - value(const value& rhs); - value& operator=(value rhs); - value(const std::string& pkey, const value& rhs); - - value(const std::initializer_list& i); - value(const std::vector& v, bool array_on_empty = true); - value(const std::unordered_map& m); - value(const std::string& pkey, const std::vector& v, bool array_on_empty = true); - value(const std::string& pkey, const std::unordered_map& m); - value(const std::string& pkey, std::nullptr_t); - value(std::nullptr_t); - - value(const char* i); - value(const std::string& pkey, const char* i); - -#define MIGRAPHX_VALUE_GENERATE_DECL_METHODS(vt, cpp_type) \ - value(cpp_type i); \ - value(const std::string& pkey, cpp_type i); \ - value& operator=(cpp_type rhs); \ - bool is_##vt() const; \ - const cpp_type& get_##vt() const; \ - const cpp_type* if_##vt() const; - MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_DECL_METHODS) - - template - using literal_to_string = std::conditional_t<(std::is_convertible{} and - std::is_convertible{}), - std::string, - T>; - - template - using pick_numeric = std::conditional_t< - std::is_floating_point{}, - double, - std::conditional_t{}, - std::int64_t, - std::conditional_t{}, std::uint64_t, T>>>; - - template - using pick = pick_numeric{}, - std::underlying_type, - std::enable_if>::type>; - - template - using is_pickable = - bool_c<((std::is_arithmetic{} or std::is_enum{}) and not std::is_pointer{})>; - - template - using range_value = std::decay_t().end(), *std::declval().begin())>; - - template - using is_generic_range = - bool_c<(std::is_convertible, value>{} and - not std::is_convertible{} and not std::is_convertible{})>; - - template {})> - value(const T& r) : value(from_values(r)) - { - } - - template {})> - value(const std::string& pkey, const T& r) : value(pkey, from_values(r)) - { - } - - template {})> - value(T i) : value(static_cast>(i)) - { - } - template {})> - value(const std::string& pkey, T i) : value(pkey, static_cast>(i)) - { - } - template - value(const std::pair& p) : value(p.first, p.second) - { - } - template {})> - value& operator=(T rhs) - { - return *this = static_cast>(rhs); // NOLINT - } - template {})> - value& operator=(T rhs) - { - return *this = from_values(rhs); // NOLINT - } - - value& operator=(const char* c); - value& operator=(std::nullptr_t); - value& operator=(const std::initializer_list& i); - - bool is_array() const; - const std::vector& get_array() const; - const std::vector* if_array() const; - - bool is_object() const; - const std::vector& get_object() const; - const std::vector* if_object() const; - - bool is_null() const; - - const std::string& get_key() const; - value* find(const std::string& pkey); - const value* find(const std::string& pkey) const; - bool contains(const std::string& pkey) const; - std::size_t size() const; - bool empty() const; - const value* data() const; - value* data(); - value* begin(); - const value* begin() const; - value* end(); - const value* end() const; - - value& front(); - const value& front() const; - value& back(); - const value& back() const; - value& at(std::size_t i); - const value& at(std::size_t i) const; - value& at(const std::string& pkey); - const value& at(const std::string& pkey) const; - value& operator[](std::size_t i); - const value& operator[](std::size_t i) const; - value& operator[](const std::string& pkey); - - void clear(); - void resize(std::size_t n); - void resize(std::size_t n, const value& v); - - std::pair insert(const value& v); - value* insert(const value* pos, const value& v); - - template - std::pair emplace(Ts&&... xs) - { - return insert(value(std::forward(xs)...)); - } - - template - value* emplace(const value* pos, Ts&&... xs) - { - return insert(pos, value(std::forward(xs)...)); - } - - void push_back(const value& v) { insert(end(), v); } - - void push_front(const value& v) { insert(begin(), v); } - - value with_key(const std::string& pkey) const; - value without_key() const; - - template - void visit(Visitor v) const - { - switch(this->get_type()) - { - case null_type: { - std::nullptr_t null{}; - if(this->key.empty()) - v(null); - else - v(std::make_pair(this->get_key(), std::ref(null))); - return; - } -#define MIGRAPHX_VALUE_GENERATE_CASE(vt, cpp_type) \ - case vt##_type: { \ - if(this->key.empty()) \ - v(this->get_##vt()); \ - else \ - v(std::make_pair(this->get_key(), std::ref(this->get_##vt()))); \ - return; \ - } - MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_CASE) - MIGRAPHX_VALUE_GENERATE_CASE(array, ) - MIGRAPHX_VALUE_GENERATE_CASE(object, ) - } - MIGRAPHX_THROW("Unknown type"); - } - - // Visit value without key - template - void visit_value(Visitor v) const - { - switch(this->get_type()) - { - case null_type: { - std::nullptr_t null{}; - v(null); - return; - } -#define MIGRAPHX_VALUE_GENERATE_CASE_VALUE(vt, cpp_type) \ - case vt##_type: { \ - v(this->get_##vt()); \ - return; \ - } - MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_CASE_VALUE) - MIGRAPHX_VALUE_GENERATE_CASE_VALUE(array, ) - MIGRAPHX_VALUE_GENERATE_CASE_VALUE(object, ) - } - MIGRAPHX_THROW("Unknown type"); - } - - template - To to() const - { - To result; - this->visit([&](auto y) { result = try_convert_value(y); }); - return result; - } - - template - literal_to_string value_or(const To& default_value) const - { - if(this->is_null()) - return default_value; - return to>(); - } - - template - std::vector to_vector() const - { - std::vector result; - const auto& values = is_object() ? get_object() : get_array(); - result.reserve(values.size()); - std::transform(values.begin(), values.end(), std::back_inserter(result), [&](auto v) { - return v.template to(); - }); - return result; - } - - template - literal_to_string get(const std::string& pkey, const To& default_value) const - { - const auto* v = find(pkey); - if(v == this->end()) - return default_value; - return v->to>(); - } - - template - std::vector get(const std::string& pkey, const std::vector& default_value) const - { - const auto* v = find(pkey); - if(v == this->end()) - return default_value; - return v->to_vector(); - } - - template - std::vector> get(const std::string& pkey, - const std::initializer_list& default_value) const - { - return get(pkey, - std::vector>{default_value.begin(), default_value.end()}); - } - - MIGRAPHX_EXPORT friend bool operator==(const value& x, const value& y); - MIGRAPHX_EXPORT friend bool operator!=(const value& x, const value& y); - MIGRAPHX_EXPORT friend bool operator<(const value& x, const value& y); - MIGRAPHX_EXPORT friend bool operator<=(const value& x, const value& y); - MIGRAPHX_EXPORT friend bool operator>(const value& x, const value& y); - MIGRAPHX_EXPORT friend bool operator>=(const value& x, const value& y); - - MIGRAPHX_EXPORT friend std::ostream& operator<<(std::ostream& os, const value& d); - - std::size_t hash() const; - - void debug_print(bool show_type = false) const; - - type_t get_type() const; - - private: - template - std::vector from_values(const T& r) - { - std::vector v; - std::transform( - r.begin(), r.end(), std::back_inserter(v), [&](auto&& e) { return value(e); }); - return v; - } - std::shared_ptr x; - std::string key; -}; - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -namespace std { -template <> -struct hash -{ - using argument_type = migraphx::value; - using result_type = std::size_t; - result_type operator()(const migraphx::value& x) const { return x.hash(); } -}; - -} // namespace std - -#endif diff --git a/docker/rocm/migraphx/include/migraphx/verify.hpp b/docker/rocm/migraphx/include/migraphx/verify.hpp deleted file mode 100644 index 937796dd6..000000000 --- a/docker/rocm/migraphx/include/migraphx/verify.hpp +++ /dev/null @@ -1,290 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_VERIFY_HPP -#define MIGRAPHX_GUARD_VERIFY_HPP - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_VERIFY_ENABLE_ALLCLOSE) -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace verify { - -// Compute the value of a range -template -using range_value = std::decay_t().begin())>; - -struct sum_fn -{ - template - auto operator()(T x, U y) const - { - return x + y; - } -}; -static constexpr sum_fn sum{}; - -struct max_fn -{ - template - static T id(T x) - { - return x; - } - - template - auto operator()(T x, U y) const - { - return x > y ? x : y; - } -}; -static constexpr max_fn max{}; - -namespace abs_diff_detail { -using std::fabs; -struct fn -{ - template - auto operator()(T x, U y) const - { - return fabs(x - y); - } -}; - -} // namespace abs_diff_detail - -static constexpr abs_diff_detail::fn abs_diff{}; - -struct not_finite_fn -{ - template - bool operator()(T x) const - { - return not std::isfinite(static_cast(x)); - } -}; -static constexpr not_finite_fn not_finite{}; - -struct compare_mag_fn -{ - template - bool operator()(T x, U y) const - { - return std::fabs(x) < std::fabs(y); - } -}; -static constexpr compare_mag_fn compare_mag{}; - -struct square_diff_fn -{ - template - double operator()(T x, U y) const - { - return (x - y) * (x - y); - } -}; -static constexpr square_diff_fn square_diff{}; - -template -bool range_empty(R1&& r1) -{ - return r1.begin() == r1.end(); -} - -template -auto range_distance(R1&& r1) -{ - return std::distance(r1.begin(), r1.end()); -} - -template -bool range_zero(R1&& r1) -{ - return std::all_of(r1.begin(), r1.end(), [](auto x) { return float_equal(x, 0); }); -} - -template -T range_product(R1&& r1, R2&& r2, T state, Reducer r, Product p) -{ - return std::inner_product(r1.begin(), r1.end(), r2.begin(), state, r, p); -} - -template -std::size_t mismatch_idx(R1&& r1, R2&& r2, Compare compare) -{ - auto p = std::mismatch(r1.begin(), r1.end(), r2.begin(), compare); - return std::distance(r1.begin(), p.first); -} - -template -long find_idx(R1&& r1, Predicate p) -{ - auto it = std::find_if(r1.begin(), r1.end(), p); - if(it == r1.end()) - return -1; - else - return std::distance(r1.begin(), it); -} - -template -double max_diff(R1&& r1, R2&& r2) -{ - return range_product(r1, r2, 0.0, max, abs_diff); -} - -template -std::size_t mismatch_diff(R1&& r1, R2&& r2, T diff) -{ - return mismatch_idx(r1, r2, [&](auto x, auto y) { - auto d = abs_diff(x, y); - return float_equal(d, diff); - }); -} - -template -double rms_range(const R1& r1, const R2& r2) -{ - std::size_t n = range_distance(r1); - if(n == range_distance(r2)) - { - double square_difference = range_product(r1, r2, 0.0, sum_fn{}, square_diff); - double mag1 = *std::max_element(r1.begin(), r1.end(), compare_mag); - double mag2 = *std::max_element(r2.begin(), r2.end(), compare_mag); - double mag = - std::max({std::fabs(mag1), std::fabs(mag2), std::numeric_limits::min()}); - return std::sqrt(square_difference) / (std::sqrt(n) * mag); - } - else - return std::numeric_limits>::max(); -} - -template -double get_rms_tol(const R&, std::size_t tolerance = 80) -{ - return std::numeric_limits>::epsilon() * tolerance; -} - -/* -C++ doesn't support named arguments, this is just wrapper that helps distinguish between actual -results v/s expected results arguments. -*/ -template -struct expected -{ - expected() = default; - explicit expected(const T& input) : x(&input) {} - const T& data() const - { - assert(x != nullptr); - return *x; - } - - private: - const T* x = nullptr; -}; - -// deduction guide for templated expected class -template -expected(const T&) -> expected; - -struct tolerance -{ - double rms_tol = 0.001; - double atol = 0.001; - double rtol = 0.001; -}; - -/* -MIGraphX implementation of numpy's np.allclose() which checks if elementwise absolute diff is within -tolerance using this formula: abs(a - b) < atol + rtol(abs(b)) -*/ -template -bool allclose(const R1& r1, const R2& r2, tolerance tols) -{ - std::size_t n = range_distance(r1); - if(n == range_distance(r2)) - { - auto idx = mismatch_idx(r1, r2, [&](auto x, auto y) { - return abs_diff(double(x), double(y)) < tols.atol + tols.rtol * std::abs(double(y)); - }); - return idx >= range_distance(r1); - } - return false; -} - -template -bool verify_rms_range(const R1& r1, - const R2& r2, - std::size_t tolerance = 80, - double* out_rms_error = nullptr) -{ - double threshold = get_rms_tol(r1, tolerance); - auto error = rms_range(r1, r2); - if(out_rms_error != nullptr) - *out_rms_error = error; - return error <= threshold; -} - -template -bool verify_range_with_tolerance(const R1& r1, - const expected& r2, - tolerance tols = tolerance{}, - double* out_rms_error = nullptr) -{ - auto rms_error = rms_range(r1, r2.data()); - // disable ewise_verify by default for now, it requires lot of tests to be fixed - bool ewise_verify = true; - if(enabled(MIGRAPHX_VERIFY_ENABLE_ALLCLOSE{})) - { - ewise_verify = allclose(r1, r2.data(), tols); - } - if(out_rms_error != nullptr) - *out_rms_error = rms_error; - return rms_error <= tols.rms_tol and ewise_verify; -} - -// expected argument should be passed as second, but if it is passed as the first by mistake then -// flip the order -template -bool verify_range_with_tolerance(const expected& r1, - const R2& r2, - tolerance tols = tolerance{}, - double* out_rms_error = nullptr) -{ - return verify_rms_range(r2, r1, tols, out_rms_error); -} - -} // namespace verify -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/include/migraphx/verify_args.hpp b/docker/rocm/migraphx/include/migraphx/verify_args.hpp deleted file mode 100644 index 91166af2e..000000000 --- a/docker/rocm/migraphx/include/migraphx/verify_args.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_VERIFY_ARGS_HPP -#define MIGRAPHX_GUARD_RTGLIB_VERIFY_ARGS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_EXPORT bool verify_args(const std::string& name, - const argument& target_arg, - const verify::expected& ref_arg, - verify::tolerance); - -MIGRAPHX_EXPORT bool verify_args_with_tolerance(const std::string& name, - const argument& target_arg, - const verify::expected& ref_arg, - std::size_t tolerance = 80); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/inline_module.cpp b/docker/rocm/migraphx/inline_module.cpp deleted file mode 100644 index 775c4f80c..000000000 --- a/docker/rocm/migraphx/inline_module.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static void inline_submodule(module& m, instruction_ref ins, bool cond) -{ - const auto& mod_inputs = ins->module_inputs(); - module_ref smod = cond ? mod_inputs.at(0) : mod_inputs.at(1); - auto mod_outputs = m.insert_instructions(ins, smod); - - auto ins_outputs = ins->outputs(); - assert(mod_outputs.size() >= ins_outputs.size()); - for(const auto& out : ins_outputs) - { - auto val = out->get_operator().to_value(); - assert(val.contains("index")); - auto index = val.at("index").to(); - m.replace_instruction(out, mod_outputs.at(index)); - } -} - -void inline_module::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "if") - continue; - - auto arg_cond = ins->inputs().front()->eval(); - if(not arg_cond.empty()) - { - bool cond = arg_cond.at(); - inline_submodule(m, ins, cond); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/insert_pad.cpp b/docker/rocm/migraphx/insert_pad.cpp deleted file mode 100644 index a46ccbe9b..000000000 --- a/docker/rocm/migraphx/insert_pad.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static void update_op(const instruction_ref& input, const instruction_ref& ins, module& m) -{ - auto op = ins->get_operator(); - auto val = op.to_value(); - auto op_padding = val.at("padding").to_vector(); - - // skip if shape is dynamic - if(input->get_shape().dynamic()) - { - return; - } - - auto kdims = input->get_shape().lens().size() - 2; - if(std::equal(op_padding.begin(), - op_padding.begin() + kdims, - op_padding.begin() + kdims, - op_padding.end())) - return; - - std::vector padding(input->get_shape().lens().size() * 2, 0); - std::vector pads_l(op_padding.begin(), op_padding.begin() + kdims); - std::vector pads_r(op_padding.begin() + kdims, op_padding.end()); - op_padding = std::vector(kdims * 2, 0); - op.from_value({{"padding", op_padding}}); - - std::copy(pads_l.begin(), pads_l.end(), padding.begin() + 2); - std::copy(pads_r.begin(), pads_r.end(), padding.begin() + kdims + 2 + 2); - - auto pad_op = m.insert_instruction(ins, op::pad{padding}, input); - - auto new_inputs = ins->inputs(); - new_inputs.front() = pad_op; - - m.replace_instruction(ins, op, new_inputs); -} - -static void update_pooling(const instruction_ref& input, const instruction_ref& ins, module& m) -{ - auto op = any_cast(ins->get_operator()); - if(op.mode == op::pooling_mode::average) - { - return; - } - auto kdims = input->get_shape().ndim() - 2; - if(std::equal(op.padding.begin(), - op.padding.begin() + kdims, - op.padding.begin() + kdims, - op.padding.end())) - return; - - std::vector padding(input->get_shape().ndim() * 2, 0); - std::vector pads_l(op.padding.begin(), op.padding.begin() + kdims); - std::vector pads_r(op.padding.begin() + kdims, op.padding.end()); - op.padding = std::vector(kdims * 2, 0); - std::copy(pads_l.begin(), pads_l.end(), padding.begin() + 2); - std::copy(pads_r.begin(), pads_r.end(), padding.begin() + kdims + 2 + 2); - - float pad_val = 0.0f; // for the lpnorm - if(op.mode == op::pooling_mode::max) - { - // maxpool uses lowest value for padding - pad_val = std::numeric_limits::lowest(); - } - auto pad_op = m.insert_instruction(ins, op::pad{padding, pad_val}, input); - - auto new_inputs = ins->inputs(); - new_inputs.front() = pad_op; - - m.replace_instruction(ins, op, new_inputs); -} - -void insert_pad::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - const std::string& op_name = ins->name(); - if(not contains(ops, op_name)) - continue; - auto input = ins->inputs().front(); - if(op_name == "convolution" or op_name == "im2col") - update_op(input, ins, m); - else if(op_name == "pooling") - update_pooling(input, ins, m); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/instruction.cpp b/docker/rocm/migraphx/instruction.cpp deleted file mode 100644 index 9c2d59f97..000000000 --- a/docker/rocm/migraphx/instruction.cpp +++ /dev/null @@ -1,564 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -auto equal_to(const T& x) -{ - return [&](const T& y) { return std::equal_to{}(x, y); }; -} - -instruction::instruction(operation o, shape r, std::vector args) - : op(std::move(o)), result(std::move(r)), arguments(std::move(args)) -{ -} - -instruction::instruction(operation o, - shape r, - std::vector args, - std::vector modules) - : op(std::move(o)), - result(std::move(r)), - arguments(std::move(args)), - module_args(std::move(modules)) -{ -} - -instruction::instruction(literal l) - : op(builtin::literal{}), result(l.get_shape()), lit(std::move(l)) -{ -} - -struct replace_shape_order -{ - instruction_ref start; - - std::size_t location(instruction_ref x) const { return std::distance(start, x); } - - bool operator()(instruction_ref x, instruction_ref y) const - { - return location(x) > location(y); - } -}; - -void instruction::replace(const shape& r) -{ - if(r != result) - { - result = r; - if(output.empty()) - { - return; - } - auto start = std::find_if(output.front()->inputs().begin(), - output.front()->inputs().end(), - [&](instruction_ref x) { return this == as_address(x); }); - assert(as_address(*start) == this); - std::priority_queue, replace_shape_order> q( - output.begin(), output.end(), replace_shape_order{*start}); - while(not q.empty()) - { - instruction_ref ins = q.top(); - q.pop(); - assert(ins->name() == "@return" or ins->name().front() != '@'); - shape new_r = compute_shape(ins->op, ins->arguments, ins->module_args); - if(new_r != ins->result) - { - ins->result = new_r; - std::copy(ins->output.begin(), ins->output.end(), migraphx::push_inserter(q)); - } - } - } -} - -void instruction::replace(operation o) -{ - normalized = false; - op = std::move(o); - recompute_shape(); -} - -void instruction::recompute_shape() { replace(compute_shape(op, arguments, module_args)); } - -void instruction::clear_arguments() -{ - for(auto&& arg : arguments) - { - arg->remove_output(*this); - } - arguments.clear(); - module_args.clear(); -} - -bool operator==(const instruction& i, instruction_ref ref) -{ - return std::addressof(i) == std::addressof(*ref); -} - -bool instruction::valid(instruction_ref start, bool check_order) const -{ - return valid() and std::all_of(arguments.begin(), arguments.end(), [&](instruction_ref i) { - auto self = std::find(i->outputs().begin(), i->outputs().end(), *this); - bool ret = self != i->outputs().end(); - if(check_order) - { - // check arguments for this instruction before this instruction - ret = ret and (std::distance(start, i) < std::distance(start, *self)); - } - return ret; - }); -} - -bool instruction::valid() const -{ - shape computed; - if(op.name() == "@literal") - { - computed = lit.get_shape(); - } - else if(op.name() == "@param") - { - computed = result; - } - else - { - try - { - computed = compute_shape(op, arguments, module_args); - } - catch(migraphx::exception&) - { - return false; - } - } - - return (result == computed) and - std::all_of(output.begin(), output.end(), [&](instruction_ref i) { - return std::find(i->inputs().begin(), i->inputs().end(), *this) != i->inputs().end(); - }); -} - -shape instruction::get_shape() const { return result; } - -const literal& instruction::get_literal() const -{ - assert(op.name() == "@literal"); - return lit; -} - -const operation& instruction::get_operator() const { return op; } - -std::string instruction::name() const { return op.name(); } - -const std::vector& instruction::inputs() const { return arguments; } - -const std::vector& instruction::module_inputs() const { return module_args; } - -const std::vector& instruction::outputs() const { return output; } - -bool operator==(const instruction& x, const instruction& y) -{ - if(not std::equal(x.arguments.begin(), - x.arguments.end(), - y.arguments.begin(), - y.arguments.end(), - std::equal_to{})) - return false; - if(std::tie(x.result, x.op, x.module_args) != std::tie(y.result, y.op, y.module_args)) - return false; - if(x.name() == "@literal") - return x.lit == y.lit; - return true; -} - -bool operator!=(const instruction& x, const instruction& y) { return not(x == y); } - -bool operator==(instruction_ref ref, const instruction& i) { return i == ref; } - -bool operator!=(const instruction& i, instruction_ref ref) { return not(i == ref); } - -bool operator!=(instruction_ref ref, const instruction& i) { return not(i == ref); } - -void instruction::add_output(instruction_ref ins) -{ - if(std::find_if(output.begin(), output.end(), equal_to(ins)) == output.end()) - output.push_back(ins); -} - -void instruction::backreference(instruction_ref ref) -{ - for(auto&& arg : ref->inputs()) - arg->add_output(ref); -} - -void instruction::replace_argument(instruction_ref ins, - instruction_ref old, - instruction_ref new_ins) -{ - ins->replace_argument(old, new_ins); - backreference(ins); - ins->recompute_shape(); -} - -void instruction::replace_mod_argument(instruction_ref ins, module_ref old, module_ref new_mod) -{ - ins->replace_mod_argument(old, new_mod); - backreference(ins); - ins->recompute_shape(); -} - -void instruction::replace(instruction_ref ins, - operation o, - const shape& r, - std::vector args) -{ - ins->replace(std::move(o), r, std::move(args)); - backreference(ins); -} - -void instruction::replace(instruction_ref ins, - operation o, - const shape& r, - std::vector args, - std::vector module_args) -{ - ins->replace(std::move(o), r, std::move(args), std::move(module_args)); - backreference(ins); -} - -void instruction::replace(operation o, const shape& r, std::vector args) -{ - normalized = false; - op = std::move(o); - replace(r); - replace(std::move(args)); -} - -void instruction::replace(operation o, - const shape& r, - std::vector args, - std::vector mdl_args) -{ - op = std::move(o); - replace(r); - replace(std::move(args), std::move(mdl_args)); -} - -void instruction::replace_refs( - instruction_ref ins, - const std::unordered_map& map_insts, - const std::unordered_map& map_mods) -{ - const auto& args = ins->inputs(); - for(const auto& arg : args) - { - if(contains(map_insts, arg)) - { - instruction::replace_argument(ins, arg, map_insts.at(arg)); - } - } - - const auto& module_args = ins->module_inputs(); - if(module_args.empty()) - return; - - for(const auto& mod : module_args) - { - if(contains(map_mods, mod)) - { - instruction::replace_mod_argument(ins, mod, map_mods.at(mod)); - } - } -} - -void instruction::replace(std::vector args) -{ - clear_arguments(); - arguments = std::move(args); -} - -void instruction::replace(std::vector args, std::vector mdl_args) -{ - clear_arguments(); - arguments = std::move(args); - module_args = std::move(mdl_args); -} - -void instruction::replace_argument(instruction_ref old, instruction_ref new_ins) -{ - assert(std::any_of(arguments.begin(), arguments.end(), equal_to(old))); - std::replace_if(arguments.begin(), arguments.end(), equal_to(old), new_ins); - old->remove_output(*this); -} - -void instruction::replace_mod_argument(module_ref old, module_ref new_mod) -{ - assert(std::any_of(module_args.begin(), module_args.end(), [&](auto i) { return i == old; })); - std::replace(module_args.begin(), module_args.end(), old, new_mod); -} - -bool instruction::is_undefined() const -{ - if(op.name() == "undefined") - { - return true; - } - else if(this->inputs().empty()) - { - return false; - } - else - { - return std::all_of(this->inputs().begin(), this->inputs().end(), [](auto arg) { - return arg->is_undefined(); - }); - } -} - -bool instruction::can_eval() const -{ - if(op.name() == "@literal") - { - return true; - } - else if(is_context_free(op)) - { - return std::all_of( - this->inputs().begin(), this->inputs().end(), [](auto arg) { return arg->can_eval(); }); - } - else - { - return false; - } -} - -argument instruction::eval(bool check_eval) const -{ - if(op.name() == "@literal") - { - return this->get_literal().get_argument(); - } - if(is_context_free(op)) - { - if(check_eval and not this->can_eval()) - return {}; - std::vector args; - std::transform(this->inputs().begin(), - this->inputs().end(), - std::back_inserter(args), - [](auto arg) { return arg->eval(false); }); - return normalized_operator().compute(result, args); - } - return {}; -} - -void instruction::finalize(context& ctx) -{ - if(has_finalize(this->op)) - this->op.finalize(ctx, this->get_shape(), to_shapes(this->inputs())); -} - -void instruction::print(std::ostream& os, - instruction_ref ins, - const std::unordered_map& names) -{ - os << names.at(ins) << " = "; - - os << ins->get_operator(); - - if(ins->name() == "@literal") - { - if(ins->get_literal().get_shape().elements() > 10) - os << "{ ... }"; - else - os << "{" << ins->get_literal() << "}"; - } - - if(not ins->inputs().empty()) - { - char delim = '('; - for(auto&& arg : ins->inputs()) - { - std::string arg_name = contains(names, arg) ? names.at(arg) : "?"; - os << delim << arg_name; - delim = ','; - } - os << ")"; - } - - // print module inputs - if(not ins->module_inputs().empty()) - { - std::string delim = ", ["; - for(const const_module_ref& mod_arg : ins->module_inputs()) - { - os << delim << mod_arg->name(); - delim = ", "; - } - os << "]"; - } - - // skip return instruction shape - if(ins->name() != "@return") - os << " -> " << ins->get_shape(); - - // print tid - if(ins->target_id != 0) - os << ", target_id=" << ins->target_id; -} - -static void debug_name(std::ostream& os, const instruction& ins) -{ - if(ins.name() == "@literal") - { - os << "@literal"; - if(ins.get_literal().get_shape().elements() > 10) - os << "{ ... }"; - else - os << "{" << ins.get_literal() << "}"; - } - else - { - os << ins.get_operator(); - } -} - -void instruction::debug_print() const -{ - debug_name(std::cout, *this); - std::string delim = "("; - for(auto arg : this->inputs()) - { - std::cout << delim; - debug_name(std::cout, *arg); - delim = ", "; - } - if(not this->inputs().empty()) - std::cout << ")"; - std::cout << " -> " << this->get_shape() << std::endl; -} - -instruction_ref instruction::get_output_alias(instruction_ref ins, bool shallow) -{ - auto i = ins->get_operator().output_alias(to_shapes(ins->inputs())); - if(i < 0) - return ins; - if(shallow) - return ins->inputs().at(i); - return get_output_alias(ins->inputs().at(i)); -} - -void instruction::set_normalized(bool value) { normalized = value; } - -bool instruction::is_normalized() const { return normalized; } - -bool instruction::need_normalization() const -{ - return this->get_operator().need_normalization() and not normalized; -} - -operation instruction::normalized_operator() const -{ - operation o = this->get_operator(); - if(this->need_normalization()) - { - auto s = this->inputs().front()->get_shape(); - if(not normalize_attributes(o, s)) - return this->get_operator(); - } - return o; -} -std::size_t instruction::get_target_id() const { return target_id; } - -void instruction::set_target_id(std::size_t tid) { this->target_id = tid; } - -std::vector to_shapes(const std::vector& args) -{ - std::vector shapes(args.size()); - std::transform( - args.begin(), args.end(), shapes.begin(), [](instruction_ref i) { return i->get_shape(); }); - return shapes; -} - -shape compute_shape(const operation& op, const std::vector& args) -{ - return op.compute_shape(to_shapes(args)); -} - -shape compute_shape(const operation& op, - const std::vector& args, - const std::vector& mods) -{ - if(mods.empty()) - { - return op.compute_shape(to_shapes(args)); - } - else - { - return op.compute_shape(to_shapes(args), mods); - } -} - -std::vector try_compute_shape(const operation& op, const std::vector& inputs) -{ - shape new_shape; - try - { - new_shape = op.compute_shape(inputs); - } - catch(...) - { - return {}; - } - return {new_shape}; -} - -migraphx::instruction* as_address(const instruction_ref& ins) noexcept -{ - return std::addressof(*ins); -} - -bool reaches(instruction_ref start, instruction_ref end) -{ - std::unordered_set visited; - return fix([&](auto self, auto ins) -> bool { - if(ins == start) - return true; - if(not visited.insert(ins).second) - return false; - return std::any_of(ins->inputs().begin(), ins->inputs().end(), self); - })(end); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/json.cpp b/docker/rocm/migraphx/json.cpp deleted file mode 100644 index aeda7a0af..000000000 --- a/docker/rocm/migraphx/json.cpp +++ /dev/null @@ -1,177 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using json = nlohmann::json; - -void value_to_json(const value& val, json& j); -migraphx::value value_from_json(const json& j); - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -namespace nlohmann { -template <> -struct adl_serializer -{ - static void to_json(json& j, const migraphx::value& val) { migraphx::value_to_json(val, j); } - - static void from_json(const json& j, migraphx::value& val) - { - val = migraphx::value_from_json(j); - } -}; -} // namespace nlohmann - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using json = nlohmann::json; - -template -void value_to_json(const T& x, json& j) -{ - j = x; -} - -void value_to_json(const value::binary& x, json& j) -{ - j = json::object(); - j["bytes"] = std::vector(x.begin(), x.end()); -} - -void value_to_json(const std::vector& x, json& j) -{ - for(const auto& v : x) - { - if(v.get_key().empty()) - { - j.push_back(v); - } - else - { - j[v.get_key()] = v.without_key(); - } - } -} - -void value_to_json(std::nullptr_t&, json& j) { j = {}; } - -void value_to_json(const value& val, json& j) -{ - if(val.is_array()) - { - j = json::array(); - } - - if(val.is_object()) - { - j = json::object(); - } - - val.visit([&](auto v) { value_to_json(v, j); }); -} - -migraphx::value value_from_json(const json& j) -{ - migraphx::value val; - json::value_t type = j.type(); - switch(type) - { - case json::value_t::null: val = nullptr; break; - - case json::value_t::boolean: val = j.get(); break; - - case json::value_t::number_float: val = j.get(); break; - - case json::value_t::number_integer: val = j.get(); break; - - case json::value_t::number_unsigned: val = j.get(); break; - - case json::value_t::string: val = j.get(); break; - - case json::value_t::array: - val = migraphx::value::array{}; - std::transform(j.begin(), j.end(), std::back_inserter(val), [&](const json& jj) { - return jj.get(); - }); - break; - - case json::value_t::object: - if(j.contains("bytes") and j.size() == 1) - { - val = migraphx::value::binary{j["bytes"].get>()}; - } - else - { - val = migraphx::value::object{}; - for(const auto& item : j.items()) - { - const auto& key = item.key(); - const json& jv = item.value(); - val[key] = jv.get(); - } - } - break; - - case json::value_t::binary: MIGRAPHX_THROW("Convert JSON to Value: binary type not supported!"); - case json::value_t::discarded: - MIGRAPHX_THROW("Convert JSON to Value: discarded type not supported!"); - } - - return val; -} - -std::string to_json_string(const value& val) -{ - json j = val; - return j.dump(); -} - -std::string to_pretty_json_string(const value& val, std::size_t indent) -{ - json j = val; - return j.dump(indent); -} - -migraphx::value from_json_string(const char* str, std::size_t size) -{ - json j = json::parse(str, str + size); - return j.get(); -} -migraphx::value from_json_string(const std::string& str) -{ - json j = json::parse(str); - return j.get(); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/layout_convolution.cpp b/docker/rocm/migraphx/layout_convolution.cpp deleted file mode 100644 index 83acb839c..000000000 --- a/docker/rocm/migraphx/layout_convolution.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace { -std::vector get_permutation(instruction_ref ins, const layout_convolution& lc) -{ - if(lc.channels_last) - { - std::vector perm(ins->get_shape().ndim()); - std::iota(perm.begin() + 1, perm.end() - 1, 2); - perm.back() = 1; - return perm; - } - return find_permutation(ins->inputs().front()->get_shape()); -} - -bool skip_layout(const shape& s) -{ - return s.ndim() == 1 or s.dynamic() or s.type() == shape::tuple_type; -} - -void preserve_output_layout(module& m) -{ - auto last = std::prev(m.end()); - if(last->name() == "@return") - { - std::vector outputs; - std::transform(last->inputs().begin(), - last->inputs().end(), - std::back_inserter(outputs), - [&](instruction_ref ins) { - if(skip_layout(ins->get_shape())) - return ins; - auto permutation = find_permutation(ins->get_shape()); - return m.insert_instruction( - last, make_op("layout", {{"permutation", permutation}}), ins); - }); - m.replace_return(outputs); - } - else if(not skip_layout(last->get_shape())) - { - auto permutation = find_permutation(last->get_shape()); - m.add_instruction(make_op("layout", {{"permutation", permutation}}), last); - } -} - -void transform_convolutions(module& m, const layout_convolution& lc) -{ - for(auto ins : iterator_for(m)) - { - if(not contains({"convolution", "quant_convolution"}, ins->name())) - continue; - if(ins->get_shape().dynamic()) - continue; - if(ins->get_shape().lens().size() != 4) - continue; - auto v = ins->get_operator().to_value(); - if(v.at("group").to() > 1) - continue; - auto args = ins->inputs(); - auto perm = get_permutation(ins, lc); - std::transform(args.begin(), args.end(), args.begin(), [&](const auto& i) { - return m.insert_instruction(ins, make_op("layout", {{"permutation", perm}}), i); - }); - auto conv = m.insert_instruction(ins, ins->get_operator(), args); - auto c = m.insert_instruction(ins, make_op("contiguous"), conv); - m.replace_instruction(ins, c); - } -} - -void remove_layout(module& m) -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "layout") - continue; - if(ins->get_shape() != ins->inputs().front()->get_shape()) - continue; - m.replace_instruction(ins, ins->inputs().front()); - } -} -} // namespace - -void layout_convolution::apply(module_pass_manager& mpm) const -{ - preserve_output_layout(mpm.get_module()); - transform_convolutions(mpm.get_module(), *this); - mpm.run_pass(dead_code_elimination{}); - mpm.run_pass(eliminate_contiguous{"contiguous"}); - mpm.run_pass(dead_code_elimination{}); - remove_layout(mpm.get_module()); - mpm.run_pass(dead_code_elimination{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/lexing.cpp b/docker/rocm/migraphx/lexing.cpp deleted file mode 100644 index d17de0242..000000000 --- a/docker/rocm/migraphx/lexing.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -std::function lex_equal(const std::string& s) -{ - return [=](const char* start, const char* end) { - auto n = end - start; - if(n < s.size()) - return start; - if(std::equal(start, start + s.size(), s.data())) - return start + s.size(); - return start; - }; -} - -std::vector -tokenize(const char* start, const char* end, const std::vector& lexers) -{ - std::vector result; - while(start != end) - { - bool error = true; - for(const auto& l : lexers) - { - const auto* next = l(start, end); - if(next != start) - { - result.emplace_back(start, next - start); - start = next; - error = false; - break; - } - } - - if(error) - { - MIGRAPHX_THROW("TOKENIZE: no token found!"); - } - } - - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/load_save.cpp b/docker/rocm/migraphx/load_save.cpp deleted file mode 100644 index d207e5698..000000000 --- a/docker/rocm/migraphx/load_save.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -program load(const std::string& filename, const file_options& options) -{ - return load_buffer(read_buffer(filename), options); -} -program load_buffer(const std::vector& buffer, const file_options& options) -{ - return load_buffer(buffer.data(), buffer.size(), options); -} -program load_buffer(const char* buffer, std::size_t size, const file_options& options) -{ - program p; - if(options.format == "msgpack") - { - p.from_value(from_msgpack(buffer, size)); - } - else if(options.format == "json") - { - p.from_value(from_json_string(buffer, size)); - } - else - { - MIGRAPHX_THROW("Unknown format: " + options.format); - } - return p; -} - -void save(const program& p, const std::string& filename, const file_options& options) -{ - write_buffer(filename, save_buffer(p, options)); -} - -// MIOpen doesn't support serializing fusion plans with Find-2.0 APIs -void print_miopen_warning(const program& p) -{ - auto mods = p.get_modules(); - if(std::any_of(mods.begin(), mods.end(), [](const auto* m) { - return std::any_of(m->begin(), m->end(), [](const instruction& i) { - return i.name() == "gpu::miopen_fusion"; - }); - })) - { - std::cout << "[WARNING]: Program has miopen_fusion instructions for which tuned solutions " - "are not stored inside serialized MIGraphX program. Consider serializing with " - "MIGRAPHX_DISABLE_MIOPEN_FUSION=1 flag set." - << std::endl; - ; - } -} - -std::vector save_buffer(const program& p, const file_options& options) -{ - value v = p.to_value(); - print_miopen_warning(p); - std::vector buffer; - if(options.format == "msgpack") - { - buffer = to_msgpack(v); - } - else if(options.format == "json") - { - std::string s = to_json_string(v); - buffer = std::vector(s.begin(), s.end()); - } - else - { - MIGRAPHX_THROW("Unknown format: " + options.format); - } - return buffer; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/make_op.cpp b/docker/rocm/migraphx/make_op.cpp deleted file mode 100644 index a185b15c8..000000000 --- a/docker/rocm/migraphx/make_op.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -operation make_op(const std::string& name) { return load_op(name); } - -template -operation make_op_generic(const std::string& name, F for_each) -{ - auto op = load_op(name); - // Merge values - value w = op.to_value(); - for_each([&](const auto& key, const auto& x) { - if(not w.contains(key)) - // NOLINTNEXTLINE(performance-inefficient-string-concatenation) - MIGRAPHX_THROW("No key '" + key + "' in " + name); - w.at(key) = x; - }); - op.from_value(w); - return op; -} - -operation make_op(const std::string& name, - const std::initializer_list>& v) -{ - return make_op_generic(name, [&](auto f) { - for(auto&& [key, x] : v) - f(key, x); - }); -} - -operation make_op_from_value(const std::string& name, const value& v) -{ - if(not(v.is_object() or (v.empty() and v.is_array()))) - MIGRAPHX_THROW("Value is not an object for make_op: " + name); - return make_op_generic(name, [&](auto f) { - for(auto&& x : v) - f(x.get_key(), x.without_key()); - }); -} - -operation make_json_op(const std::string& name, const std::string& s) -{ - return make_op(name, from_json_string(convert_to_json(s))); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/memory_coloring.cpp b/docker/rocm/migraphx/memory_coloring.cpp deleted file mode 100644 index c8df8fa7b..000000000 --- a/docker/rocm/migraphx/memory_coloring.cpp +++ /dev/null @@ -1,372 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DEBUG_MEMORY_COLORING); - -using instruction_set = std::unordered_set; -using instruction_set_map = std::unordered_map; - -// This will build the conflict table or interference graph. This is -// essentially a map from one instruction to a set of instruction that are -// used together. Each instruction will be the allocation instruction. -instruction_set_map build_conflict_table(const module& m, std::string allocation_op) -{ - instruction_set_map conflict_table; - liveness(m, [&](auto ins, auto live_set) { - // Skip variables that aren't allocations - if(ins->name() != allocation_op) - return; - // Skip zero allocations - if(ins->get_shape().bytes() == 0) - return; - conflict_table[ins]; - for(auto i : live_set) - { - if(i == ins) - continue; - // Skip variables that aren't allocations - if(i->name() != allocation_op) - continue; - // Skip zero allocations - if(i->get_shape().bytes() == 0) - continue; - conflict_table[i].insert(ins); - conflict_table[ins].insert(i); - } - }); - assert(std::all_of(conflict_table.begin(), conflict_table.end(), [](auto&& pp) { - return pp.second.count(pp.first) == 0; - })); - return conflict_table; -} - -// Check if intervals overlap -bool is_overlap(std::pair x, std::pair y) -{ - return std::max(x.first, y.first) < std::min(x.second, y.second); -} - -struct allocation_segment -{ - using segment = std::pair; - std::unordered_map ins2segment; - - const segment* add_segment(instruction_ref ins, segment s) { return &(ins2segment[ins] = s); } - - const segment* get_segment(instruction_ref ins) const - { - auto it = ins2segment.find(ins); - if(it == ins2segment.end()) - return nullptr; - return &it->second; - } - - // Remove segment for an instruction - void remove(instruction_ref ins) - { - auto it = ins2segment.find(ins); - if(it != ins2segment.end()) - { - ins2segment.erase(it); - } - } - - std::size_t max() - { - std::size_t n = 0; - for(auto&& pp : ins2segment) - { - auto seg = pp.second; - n = std::max(n, seg.second); - } - return n; - } - - template - static bool overlaps(Iterator first, Iterator last, const segment& s) - { - return std::any_of(first, last, [&](auto&& t) { return is_overlap(s, t); }); - } - - static bool overlaps(const std::set& segments, const segment& s) - { - return overlaps(segments.begin(), segments.end(), s); - } - - static auto find_gap(const std::set& segments, std::size_t n) - { - std::size_t max_end = 0; - return std::adjacent_find(segments.begin(), segments.end(), [&](segment x, segment y) { - if(x.second < max_end) - return false; - max_end = x.second; - if(is_overlap(x, y)) - return false; - assert(y.first >= x.second); - auto k = y.first - x.second; - return (k >= n); - }); - } - - static std::size_t max_type_size(const shape& s) - { - return std::accumulate( - s.sub_shapes().begin(), - s.sub_shapes().end(), - s.type_size(), - [](auto size, const auto& sub) { return std::max(size, max_type_size(sub)); }); - } - - static std::size_t compute_alignment(instruction_ref ins) - { - auto alignment = max_type_size(ins->get_shape()); - // A rough estimate for the total number of elements - auto n = ins->get_shape().bytes() / alignment; - // Check for vectorized alignment - if(n > 4) - { - auto d = n % 4; - if(d == 0) - alignment *= 4; - if(d == 2) - alignment *= 2; - } - return alignment; - } - - static segment - next_segment(std::set& segments, instruction_ref ins, std::size_t alignment) - { - assert(ins->get_shape().bytes() > 0); - // Compute alignment - std::size_t n = 1 + (ins->get_shape().bytes() - 1) / alignment; - assert(n > 0); - std::size_t start = 0; - // Insert at end if it cant fit at the begining - if(segments.empty() or segments.begin()->first <= n) - { - auto it = find_gap(segments, n); - if(it == segments.end()) - it = std::max_element(segments.begin(), segments.end(), [&](segment x, segment y) { - return x.second < y.second; - }); - if(it != segments.end()) - start = it->second; - } - auto s = segment{start, start + n}; - assert(not overlaps(segments, s)); - segments.insert(s); - return s; - } - - static std::unordered_map - create_allocation_index(const module& m, const instruction_set_map& conflict_table) - { - std::unordered_map result; - int i = 0; - for(auto ins : iterator_for(m)) - { - if(not contains(conflict_table, ins)) - continue; - result[ins] = i++; - } - return result; - } - - // Build the allocation_color class from the conflict_table - static allocation_segment - build(const module& m, const instruction_set_map& conflict_table, std::size_t alignment) - { - allocation_segment as{}; - std::vector conflict_queue; - // Add all allocations to the conflict_queue - std::transform(conflict_table.begin(), - conflict_table.end(), - std::back_inserter(conflict_queue), - [](auto&& pp) { return pp.first; }); - - auto alloc_index = create_allocation_index(m, conflict_table); - - // Sort the conflict queue so we process the allocation with the most - // number of adjacent allocations first - std::sort(conflict_queue.begin(), conflict_queue.end(), by(std::greater<>{}, [&](auto x) { - return std::make_tuple( - conflict_table.at(x).size(), x->get_shape().bytes(), alloc_index.at(x)); - })); - // Process the conflict_queue, we refer to the current allocation as - // the parent and the adjacent allocations as children - for(auto parent : conflict_queue) - { - // Sort children by size - std::vector children(conflict_table.at(parent).begin(), - conflict_table.at(parent).end()); - std::sort(children.begin(), children.end(), by(std::less<>{}, [&](auto x) { - return std::make_tuple(x->get_shape().bytes(), alloc_index.at(x)); - })); - assert(not contains(children, parent)); - // This set is to track the segments already processed - std::set segments; - // Add all segments for the children to the segments already processed - transform_if( - children.begin(), - children.end(), - std::inserter(segments, segments.begin()), - [&](auto child) { return as.get_segment(child); }, - [&](auto child) { return *as.get_segment(child); }); - - assert(as.get_segment(parent) == nullptr); - as.add_segment(parent, next_segment(segments, parent, alignment)); - } - // Reduce the number of segments - for(std::size_t n = 0; n < 3; n++) - { - for(auto parent : conflict_queue) - { - auto children = conflict_table.at(parent); - // This set is to track the segments already processed - std::set segments; - // Add all segments for the children to the segments already processed - transform_if( - children.begin(), - children.end(), - std::inserter(segments, segments.begin()), - [&](auto child) { return as.get_segment(child); }, - [&](auto child) { return *as.get_segment(child); }); - // Get the segment for the parent - const auto* parent_segment = as.get_segment(parent); - assert(parent_segment != nullptr); - - auto s = next_segment(segments, parent, alignment); - if(s != *parent_segment and s.second <= as.max()) - { - as.add_segment(parent, s); - } - } - } - return as; - } -}; - -static std::size_t find_max_alignment(const module& m, const std::string& allocation_op) -{ - std::size_t alignment = 1; - for(auto ins : iterator_for(m)) - { - if(ins->name() != allocation_op) - continue; - alignment = std::max(allocation_segment::compute_alignment(ins), alignment); - } - return alignment; -} - -void memory_coloring::apply(module& m) const -{ - const std::size_t alignment = find_max_alignment(m, allocation_op); - auto conflict_table = build_conflict_table(m, allocation_op); - auto as = allocation_segment::build(m, conflict_table, alignment); - - // All allocations should have a segment - assert(std::all_of(conflict_table.begin(), conflict_table.end(), [&](auto&& pp) { - return as.get_segment(pp.first); - })); - - // Adjacent allocations should not have overlapping segments - assert(std::none_of(conflict_table.begin(), conflict_table.end(), [&](auto&& pp) { - auto* x = as.get_segment(pp.first); - return std::any_of(pp.second.begin(), pp.second.end(), [&](auto ins) { - auto* y = as.get_segment(ins); - assert(x and y); - return is_overlap(*x, *y); - }); - })); - - // Print out segments - if(enabled(MIGRAPHX_DEBUG_MEMORY_COLORING{})) - { - for(auto&& pp : conflict_table) - { - std::cout << "------- conflict -------" << std::endl; - auto s1 = as.ins2segment.at(pp.first); - std::cout << s1.first << ", " << s1.second << ": "; - m.debug_print(pp.first); - for(auto ins : pp.second) - { - auto s2 = as.ins2segment.at(ins); - std::cout << s2.first << ", " << s2.second << ": "; - m.debug_print(ins); - } - } - } - - // Total memory - std::size_t n = as.max() * alignment; - - // Replace allocations - auto mem = m.add_parameter("scratch", shape{shape::int8_type, {n}}); - for(auto&& [ins, seg] : as.ins2segment) - { - assert(ins->name() == allocation_op); - auto s = ins->get_shape(); - std::size_t offset = seg.first * alignment; - assert(offset < n); - m.replace_instruction( - ins, make_op("load", {{"shape", to_value(s)}, {"offset", offset}}), mem); - } - - // Replace zero allocation - for(auto ins : iterator_for(m)) - { - if(ins->name() != allocation_op) - continue; - assert(ins->get_shape().bytes() == 0); - m.replace_instruction( - ins, make_op("load", {{"shape", to_value(ins->get_shape())}, {"offset", 0}}), mem); - } - - // Remove scratch parameter if its not used - if(mem->outputs().empty()) - { - m.remove_instruction(mem); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/module.cpp b/docker/rocm/migraphx/module.cpp deleted file mode 100644 index 7e02478b3..000000000 --- a/docker/rocm/migraphx/module.cpp +++ /dev/null @@ -1,1544 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_FINALIZE) - -struct module_impl -{ - // A list is used to keep references to an instruction stable - std::list instructions; - std::unordered_set instruction_set; - std::string name; - uint32_t nparams = 0; - bool bypass = false; - bit_signal<64> changed{}; - - bool contains(instruction_ref ins) const - { - if(is_end(ins, instructions.end())) - return false; - return instruction_set.count(std::addressof(*ins)) > 0; - } - - template - instruction_ref emplace(instruction_ref pos, Ts&&... xs) - { - changed.notify(); - // cppcheck-suppress redundantInitialization - auto r = instructions.emplace(pos, std::forward(xs)...); - instruction_set.insert(std::addressof(*r)); - return r; - } - instruction_ref insert(instruction_ref pos, const instruction& ins) - { - changed.notify(); - return emplace(pos, ins); - } - - void clear() - { - changed.notify(); - instructions.clear(); - instruction_set.clear(); - nparams = 0; - } - - void push_front(const instruction& ins) { insert(instructions.begin(), ins); } - - void push_back(const instruction& ins) { insert(instructions.end(), ins); } - - template - void emplace_front(Ts&&... xs) - { - emplace(instructions.begin(), std::forward(xs)...); - } - - template - void emplace_back(Ts&&... xs) - { - emplace(instructions.end(), std::forward(xs)...); - } - - instruction_ref erase(instruction_ref pos) - { - changed.notify(); - instruction_set.erase(std::addressof(*pos)); - return instructions.erase(pos); - } - - instruction_ref erase(instruction_ref start, instruction_ref last) - { - changed.notify(); - std::for_each(start, last, [&](auto& ins) { instruction_set.erase(std::addressof(ins)); }); - return instructions.erase(start, last); - } -}; - -const operation& get_operation(instruction_ref ins) { return ins->get_operator(); } - -module::module(const std::string& name) : impl(std::make_unique()) -{ - impl->name = name; -} - -module::module(module&&) noexcept = default; -module::~module() noexcept = default; - -// copy constructor -module::module(const module& m) { assign(m); } - -// copy assignment operator -module& module::operator=(module m) -{ - std::swap(m.impl, this->impl); - return *this; -} - -std::string module::name() const { return impl->name; } - -void module::set_name(const std::string& name) { impl->name = name; } - -bool module::bypass() const { return impl->bypass; } -void module::set_bypass(bool b) { impl->bypass = b; } - -void module::assign(const module& m) -{ - // copy the impl - if(not impl) - impl = std::make_unique(); - *impl = *m.impl; - - // clear instructions - if(not impl->instructions.empty()) - { - impl->clear(); - } - - std::unordered_map ins_map; - for(auto ins : iterator_for(m)) - { - instruction_ref copy_ins{}; - if(ins->name() == "@literal") - { - auto l = ins->get_literal(); - copy_ins = impl->insert(impl->instructions.end(), instruction{l}); - } - else if(ins->name() == "@param") - { - auto&& name = any_cast(ins->get_operator()).parameter; - auto order = any_cast(ins->get_operator()).order; - auto s = ins->get_shape(); - copy_ins = impl->insert(impl->instructions.end(), - {builtin::param{name, order}, std::move(s), {}}); - impl->nparams++; - } - else if(ins->name() == "@outline") - { - auto s = ins->get_shape(); - copy_ins = impl->insert(impl->instructions.end(), {builtin::outline{s}, s, {}}); - } - else - { - // if there are sub_module inputs, need to make a copy of the submodule - auto module_args = ins->module_inputs(); - // retrieve its mapped input - auto inputs = ins->inputs(); - std::vector copy_inputs(inputs.size()); - std::transform(inputs.begin(), inputs.end(), copy_inputs.begin(), [&](auto i) { - return contains(ins_map, i) ? ins_map[i] : i; - }); - if(ins->name() == "@return") - { - copy_ins = add_return(copy_inputs); - } - else - { - copy_ins = add_instruction(ins->get_operator(), copy_inputs, module_args); - } - } - - ins_map[ins] = copy_ins; - } -} - -template -static std::vector -insert_generic_instructions_impl(module& m, - instruction_ref ins, - Range&& instructions, - std::unordered_map& map_ins, - Inserter insert) -{ - assert(m.has_instruction(ins) or is_end(ins, m.end())); - std::vector mod_outputs; - instruction_ref last; - for(instruction_ref sins : instructions) - { - last = sins; - if(contains(map_ins, sins)) - continue; - instruction_ref copy_ins; - if(sins->name() == "@literal") - { - auto l = sins->get_literal(); - copy_ins = m.add_literal(l); - } - else if(sins->name() == "@param") - { - auto&& name = any_cast(sins->get_operator()).parameter; - auto s = sins->get_shape(); - copy_ins = m.add_parameter(name, s); - } - else if(sins->name() == "@outline") - { - auto s = sins->get_shape(); - copy_ins = m.add_outline(s); - } - else - { - auto mod_args = sins->module_inputs(); - auto inputs = sins->inputs(); - std::vector copy_inputs(inputs.size()); - std::transform(inputs.begin(), inputs.end(), copy_inputs.begin(), [&](auto i) { - return contains(map_ins, i) ? map_ins[i] : i; - }); - - if(sins->name() == "@return") - { - mod_outputs = copy_inputs; - break; - } - - copy_ins = insert(m, ins, sins->get_operator(), copy_inputs, mod_args); - } - map_ins[sins] = copy_ins; - } - if(mod_outputs.empty() and instructions.begin() != instructions.end()) - mod_outputs = {map_ins.at(last)}; - return mod_outputs; -} - -template -static std::vector -insert_generic_instructions(module& m, - instruction_ref ins, - Range&& instructions, - std::unordered_map& map_ins, - module::inserter insert) -{ - if(insert == nullptr) - return insert_generic_instructions_impl( - m, ins, static_cast(instructions), map_ins, [](module& mm, auto&&... xs) { - return mm.insert_instruction(std::forward(xs)...); - }); - return insert_generic_instructions_impl( - m, ins, static_cast(instructions), map_ins, insert); -} - -instruction_ref module::add_instruction(const operation& op, std::vector args) -{ - return insert_instruction(impl->instructions.end(), op, std::move(args)); -} -instruction_ref module::insert_instruction(instruction_ref ins, - const operation& op, - std::vector args) -{ - assert(has_instruction(ins) or is_end(ins, this->end())); - assert(not starts_with(op.name(), "@")); - shape r = compute_shape(op, args); - auto result = impl->insert(ins, {op, r, std::move(args)}); - instruction::backreference(result); - assert(result->valid(begin())); - return result; -} - -instruction_ref module::add_instruction(const operation& op, - std::vector args, - std::vector module_args) -{ - return insert_instruction( - impl->instructions.end(), op, std::move(args), std::move(module_args)); -} - -instruction_ref module::insert_instruction(instruction_ref ins, - const operation& op, - std::vector args, - std::vector module_args) -{ - assert(has_instruction(ins) or is_end(ins, this->end())); - assert(not starts_with(op.name(), "@")); - auto out_shape = compute_shape(op, args, module_args); - auto result = impl->insert(ins, {op, out_shape, std::move(args), std::move(module_args)}); - instruction::backreference(result); - assert(result->valid(begin())); - return result; -} - -instruction_ref module::replace_instruction(instruction_ref ins, - const operation& op, - std::vector args) MIGRAPHX_TIDY_CONST -{ - impl->changed.notify(); - assert(has_instruction(ins)); - assert(not starts_with(op.name(), "@")); - - shape r = compute_shape(op, args); - instruction::replace(ins, op, r, std::move(args)); - assert(ins->valid(begin())); - return ins; -} - -instruction_ref module::replace_instruction(instruction_ref ins, - const operation& op, - std::vector args, - std::vector module_args) MIGRAPHX_TIDY_CONST -{ - impl->changed.notify(); - assert(has_instruction(ins)); - assert(not starts_with(op.name(), "@")); - auto out_shape = compute_shape(op, args, module_args); - instruction::replace(ins, op, out_shape, std::move(args), std::move(module_args)); - assert(ins->valid(begin())); - return ins; -} - -instruction_ref module::replace_instruction(instruction_ref ins, instruction_ref rep) -{ - impl->changed.notify(); - assert(has_instruction(ins)); - assert(ins != rep); - - if(ins == std::prev(this->end())) - { - // "rep" instruction could be used earlier in the program and moving it at the end - // may cause invalid program, therefore make an identity operation in this case. - return replace_instruction(ins, make_op("identity"), rep); - } - - // TODO: Should it be an error if the output is empty? - if(ins->outputs().empty()) - { - return rep; - } - // Make a copy of outputs which can be changed when calling replace_argument - auto outputs = ins->outputs(); - for(auto out : outputs) - { - // TODO: Check for possible cycles - if(out != rep) - { - instruction::replace_argument(out, ins, rep); - } - assert(out->valid(begin())); - } - // Replacement should not be dead code unless its the last instruction - assert(not rep->outputs().empty() or rep == std::prev(end())); - // Output of the original instruction should only be the replacement or empty - assert(ins->outputs().empty() or std::all_of(ins->outputs().begin(), - ins->outputs().end(), - [&](auto i) { return i == rep; })); - assert(ins->valid(begin())); - assert(rep->valid(begin())); - return rep; -} - -instruction_ref module::remove_instruction(instruction_ref ins) -{ - assert(has_instruction(ins)); - assert(ins->outputs().empty()); - ins->clear_arguments(); - return impl->erase(ins); -} - -instruction_ref module::remove_instructions(instruction_ref first, instruction_ref last) -{ - if(first == last) - return first; - // TODO: Check every element - assert(has_instruction(first)); - std::for_each(first, last, [&](instruction& ins) { ins.clear_arguments(); }); - assert(std::all_of(first, last, [&](const instruction& ins) { return ins.outputs().empty(); })); - return impl->erase(first, last); -} - -instruction_ref module::move_instruction(instruction_ref src, instruction_ref dst) -{ - impl->changed.notify(); - assert(has_instruction(src)); - assert(has_instruction(dst) or is_end(dst, this->end())); - impl->instructions.splice(dst, impl->instructions, src); - return src; -} - -instruction_ref module::move_instructions(instruction_ref src, instruction_ref dst) -{ - for(auto ins : src->inputs()) - { - if(not contains(this->impl->instructions, ins)) - continue; - this->move_instructions(ins, dst); - } - this->move_instruction(src, dst); - return src; -} - -std::vector -module::add_instructions(const std::vector& instructions, - std::unordered_map* map_ins, - module::inserter insert) -{ - return this->insert_instructions(this->end(), instructions, map_ins, std::move(insert)); -} - -std::vector -module::add_instructions(const_module_ref m, - std::unordered_map* map_ins, - module::inserter insert) -{ - return this->insert_instructions(this->end(), m, map_ins, std::move(insert)); -} - -std::vector -module::add_instructions(instruction_ref start, - instruction_ref last, - std::unordered_map* map_ins, - module::inserter insert) -{ - return this->insert_instructions(this->end(), start, last, map_ins, std::move(insert)); -} - -std::vector -module::insert_instructions(instruction_ref ins, - const std::vector& instructions, - std::unordered_map* map_ins, - module::inserter insert) -{ - std::unordered_map default_map_ins; - return insert_generic_instructions(*this, - ins, - instructions, - map_ins == nullptr ? default_map_ins : *map_ins, - std::move(insert)); -} - -std::vector -module::insert_instructions(instruction_ref ins, - const_module_ref m, - std::unordered_map* map_ins, - module::inserter insert) -{ - std::unordered_map default_map_ins; - return insert_generic_instructions(*this, - ins, - iterator_for(*m), - map_ins == nullptr ? default_map_ins : *map_ins, - std::move(insert)); -} - -std::vector -module::insert_instructions(instruction_ref ins, - instruction_ref start, - instruction_ref last, - std::unordered_map* map_ins, - module::inserter insert) -{ - auto r = range(start, last); - std::unordered_map default_map_ins; - return insert_generic_instructions(*this, - ins, - iterator_for(r), - map_ins == nullptr ? default_map_ins : *map_ins, - std::move(insert)); -} - -instruction_ref module::add_literal(literal l) { return insert_literal(begin(), std::move(l)); } - -instruction_ref module::add_outline(const shape& s) -{ - impl->push_front({builtin::outline{s}, s, {}}); - return impl->instructions.begin(); -} - -instruction_ref module::add_parameter(std::string name, shape s) -{ - return insert_parameter(begin(), std::move(name), std::move(s)); -} - -instruction_ref module::add_return(std::vector args) -{ - shape instr_shape = compute_shape(builtin::returns{}, args); - impl->push_back({builtin::returns{}, instr_shape, std::move(args)}); - auto result = std::prev(impl->instructions.end()); - instruction::backreference(result); - assert(result->valid(begin())); - return result; -} - -instruction_ref module::insert_literal(instruction_ref ins, literal l) -{ - impl->emplace(ins, std::move(l)); - return std::prev(ins); -} - -instruction_ref module::insert_parameter(instruction_ref ins, std::string name, shape s) -{ - assert(get_parameter_shape(name) == shape{}); - impl->insert(ins, {builtin::param{std::move(name), impl->nparams}, std::move(s), {}}); - impl->nparams++; - return std::prev(ins); -} - -instruction_ref module::replace_return(std::vector args) -{ - impl->changed.notify(); - auto last = std::prev(this->end()); - // If there is no return then add a return - if(last->name() != "@return") - return this->add_return(args); - - shape r = compute_shape(last->get_operator(), args); - instruction::replace(last, last->get_operator(), r, std::move(args)); - assert(last->valid(begin())); - - return last; -} - -shape module::get_parameter_shape(std::string name) const -{ - auto ins = std::find_if( - impl->instructions.begin(), impl->instructions.end(), [&](const instruction& x) { - if(x.name() == "@param") - { - return any_cast(x.get_operator()).parameter == name; - } - else - { - return false; - } - }); - if(ins != this->end()) - return ins->get_shape(); - else - return {}; -} - -std::vector module::get_parameter_names() const -{ - std::vector result; - std::vector params; - for(auto&& ins : impl->instructions) - { - if(ins.name() == "@param") - { - auto&& param = any_cast(ins.get_operator()); - params.push_back(param); - } - } - std::stable_sort( - params.begin(), params.end(), by(std::less<>{}, [](auto&& p) { return p.order; })); - std::transform(params.begin(), params.end(), std::back_inserter(result), [&](auto&& p) { - return p.parameter; - }); - return result; -} - -instruction_ref module::get_parameter(std::string name) const -{ - auto ins = std::find_if( - impl->instructions.begin(), impl->instructions.end(), [&](const instruction& x) { - if(x.name() == "@param") - { - return any_cast(x.get_operator()).parameter == name; - } - else - { - return false; - } - }); - if(ins != this->end()) - return ins; - else - return this->end(); -} - -std::vector module::get_parameters() const -{ - std::vector result; - auto refs = iterator_for(*this); - std::copy_if(refs.begin(), refs.end(), std::back_inserter(result), [&](instruction_ref ins) { - return ins->name() == "@param"; - }); - return result; -} - -void module::rename_parameter(instruction_ref ins, const std::string& name) -{ - impl->changed.notify(); - assert(ins->name() == "@param"); - auto op = any_cast(ins->get_operator()); - op.parameter = name; - auto outputs = ins->outputs(); - *ins = instruction{op, ins->get_shape(), {}}; - for(auto output : outputs) - ins->add_output(output); -} - -std::unordered_map module::get_parameter_shapes() const -{ - std::unordered_map result; - for(auto&& ins : impl->instructions) - { - if(ins.name() == "@param") - { - auto&& name = any_cast(ins.get_operator()).parameter; - result[name] = ins.get_shape(); - } - } - return result; -} - -bool module::has_instruction(instruction_ref ins) const { return impl->contains(ins); } - -std::size_t module::size() const { return impl->instructions.size(); } -instruction_ref module::begin() const { return impl->instructions.begin(); } -instruction_ref module::end() const { return impl->instructions.end(); } - -std::vector module::get_output_shapes() const -{ - if(impl->instructions.empty()) - return {}; - auto last_ins = impl->instructions.back(); - if(last_ins.name() == "@return") - { - const auto& output_ins = last_ins.inputs(); - std::vector output_shapes; - std::transform(output_ins.begin(), - output_ins.end(), - std::back_inserter(output_shapes), - [](auto& ins) { return ins->get_shape(); }); - - return output_shapes; - } - // The else branch is to provide backward compatibility - else - { - return {last_ins.get_shape()}; - } -} - -std::vector module::compute_shapes(const std::vector& inputs, - compute_shapes_options options) const -{ - auto params = this->get_parameter_names(); - std::sort(params.begin(), params.end()); - std::unordered_map ins_shapes; - std::unordered_map adjusted_param_shapes; - std::transform(inputs.begin(), - inputs.end(), - params.begin(), - std::inserter(adjusted_param_shapes, adjusted_param_shapes.end()), - [](auto ps, auto name) { return std::make_pair(name, ps); }); - for(auto ins : iterator_for(*this)) - { - if(ins->name() == "@param") - { - ins_shapes[ins] = - adjusted_param_shapes[any_cast(ins->get_operator()).parameter]; - if(options.strict_type and ins->get_shape().type() != ins_shapes[ins].type()) - { - MIGRAPHX_THROW(options.name + ": Mismatched type: expected " + - ins->get_shape().type_string() + " but passed " + - ins_shapes[ins].type_string()); - } - if(options.strict_lens and ins->get_shape().lens() != ins_shapes[ins].lens()) - { - MIGRAPHX_THROW(options.name + ": Mismatched lens: expected {" + - to_string_range(ins->get_shape().lens()) + "} but passed {" + - to_string_range(ins_shapes[ins].lens()) + "}"); - } - } - else if(ins->name() == "@literal") - { - if(not options.scalar_const_out_lens.empty() and ins->get_shape().scalar()) - { - std::vector strides(options.scalar_const_out_lens.size()); - ins_shapes[ins] = - shape{ins->get_shape().type(), options.scalar_const_out_lens, strides}; - } - else - { - ins_shapes[ins] = ins->get_shape(); - } - } - else - { - std::vector input_shapes; - input_shapes.resize(ins->inputs().size()); - std::transform(ins->inputs().begin(), - ins->inputs().end(), - input_shapes.begin(), - [&](auto in) { return ins_shapes.at(in); }); - if(ins->name() == "@return") - return input_shapes; - ins_shapes[ins] = ins->get_operator().compute_shape(input_shapes, ins->module_inputs()); - } - } - MIGRAPHX_THROW("No return found in the submodule"); -} - -std::vector module::compute_shapes(const std::vector& inputs) const -{ - return compute_shapes(inputs, {}); -} - -std::vector module::get_returns() const -{ - auto last = std::prev(this->end()); - if(last->name() == "@return") - return last->inputs(); - return {last}; -} - -instruction_ref module::validate() const -{ - return std::find_if( - impl->instructions.begin(), impl->instructions.end(), [&](const instruction& i) { - auto inputs = i.inputs(); - bool check_order = std::all_of( - inputs.begin(), inputs.end(), [&](auto in) { return has_instruction(in); }); - return not i.valid(impl->instructions.begin(), check_order); - }); -} - -bool is_borrowed(instruction_ref ins) -{ - auto alias = instruction::get_output_alias(ins, true); - if(alias == ins) - return false; - lifetime l = alias->get_operator().get_lifetime(); - if(l == lifetime::borrow) - return true; - return is_borrowed(alias); -} - -bool is_global(instruction_ref ins) -{ - const auto& op = instruction::get_output_alias(ins)->get_operator(); - return op.name() == "@param" or op.get_lifetime() == lifetime::global; -} - -bool is_dangling(instruction_ref ins) { return not is_global(ins) and is_borrowed(ins); } - -instruction_ref module::find_dangling_reference() const -{ - auto last = std::prev(end()); - if(last->name() == "@return") - { - auto dangling = std::find_if( - last->inputs().begin(), last->inputs().end(), [](auto x) { return is_dangling(x); }); - if(dangling != last->inputs().end()) - return *dangling; - } - else if(is_dangling(last)) - { - return last; - } - return end(); -} - -void module::finalize(std::vector& contexts) -{ - assert(not contexts.empty()); - const bool trace = enabled(MIGRAPHX_TRACE_FINALIZE{}); - for(auto ins : iterator_for(*this)) - { - if(trace) - { - std::cout << "Finalize: "; - this->debug_print(ins); - } - ins->finalize(contexts[ins->get_target_id()]); - for(const auto& smod : ins->module_inputs()) - { - smod->finalize(contexts); - } - } - - // Warn when an instruction is not normalized - auto ins = std::find_if(begin(), end(), [](auto& i) { return i.need_normalization(); }); - if(ins != end()) - std::cerr << "WARNING: Instruction needs normalization, performance may be affected." - << std::endl; -} - -std::unordered_map -module::get_ins_param_map(const std::vector& inputs, bool reverse) const -{ - std::unordered_map result; - auto params = this->get_parameters(); - assert(params.size() == inputs.size()); - sort_params(params); - if(reverse) - { - std::transform( - params.begin(), - params.end(), - inputs.begin(), - std::inserter(result, result.end()), - [&](instruction_ref param, auto input) { return std::make_pair(param, input); }); - } - else - { - std::transform( - params.begin(), - params.end(), - inputs.begin(), - std::inserter(result, result.end()), - [&](instruction_ref param, auto input) { return std::make_pair(input, param); }); - } - return result; -} - -std::vector -module::get_inputs(const std::unordered_map& map_ins) const -{ - std::vector inputs; - auto params = this->get_parameters(); - sort_params(params); - - std::transform(params.begin(), - params.end(), - std::back_inserter(inputs), - [&](instruction_ref param) { return map_ins.at(param); }); - - return inputs; -} - -static std::vector -select_params(const std::vector& instructions, - const std::unordered_map& param_map) -{ - std::vector result; - std::vector params; - std::copy_if(instructions.begin(), - instructions.end(), - std::back_inserter(params), - [&](instruction_ref ins) { return contains(param_map, ins); }); - sort_params(params); - std::transform(params.begin(), - params.end(), - std::back_inserter(result), - [&](instruction_ref ins) { return param_map.at(ins); }); - return result; -} - -static std::array -generic_split(const module& m, - const std::vector& args, - const std::vector& splits, - std::unordered_map* map_ins = nullptr) -{ - std::unordered_map param_map = - m.get_ins_param_map(args, true); - - std::unordered_set selected_instructions; - fix([&](auto self, const std::vector& inputs) { - for(auto input : inputs) - { - if(contains(selected_instructions, input)) - continue; - selected_instructions.insert(input); - self(input->inputs()); - } - })(splits); - - std::vector instructions1; - // TODO: copy_if - for(auto ins : iterator_for(m)) - { - if(not contains(selected_instructions, ins)) - continue; - instructions1.push_back(ins); - } - - std::vector inputs1 = select_params(instructions1, param_map); - module m1; - std::unordered_map map_ins1; - m1.add_instructions(instructions1, &map_ins1); - std::vector outputs; - std::transform(splits.begin(), - splits.end(), - std::back_inserter(outputs), - [&](instruction_ref ins) { return map_ins1.at(ins); }); - m1.add_return(outputs); - - std::vector instructions2; - for(auto ins : iterator_for(m)) - { - if(contains(selected_instructions, ins)) - continue; - // Input params can be used in both modules - std::vector input_params; - std::copy_if(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(input_params), - [&](instruction_ref input) { - if(input->name() != "@param") - return false; - return not contains(instructions2, input); - }); - instructions2.insert(instructions2.end(), input_params.begin(), input_params.end()); - instructions2.push_back(ins); - } - - std::vector inputs2 = splits; - module m2; - std::size_t n = 0; - std::unordered_map map_ins2; - for(auto ins : splits) - map_ins2[ins] = m2.add_parameter(param_name(n++), ins->get_shape().as_standard()); - for(auto ins : iterator_for(m)) - { - if(ins->name() != "@param") - continue; - if(not contains(instructions2, ins)) - continue; - inputs2.push_back(param_map.at(ins)); - map_ins2[ins] = m2.add_parameter(param_name(n++), ins->get_shape().as_standard()); - } - auto r = m2.add_instructions(instructions2, &map_ins2); - m2.add_return(r); - if(map_ins != nullptr) - *map_ins = map_ins2; - return {{{std::move(m1), std::move(inputs1)}, {std::move(m2), std::move(inputs2)}}}; -} - -std::array module::split(const std::vector& args, - const std::vector& splits) const -{ - return generic_split(*this, args, splits); -} - -std::array module::split(const std::vector& args, - const std::vector& splits1, - const std::vector& splits2) const -{ - std::unordered_map map_ins; - auto mods1 = generic_split(*this, args, splits1, &map_ins); - - assert(all_of(mods1[0].inputs, [&](auto ins) { return contains(args, ins); })); - assert(all_of(mods1[1].inputs, - [&](auto ins) { return contains(args, ins) or contains(splits1, ins); })); - - std::vector new_splits2; - std::transform(splits2.begin(), splits2.end(), std::back_inserter(new_splits2), [&](auto ins) { - return map_ins.at(ins); - }); - - auto mods2 = mods1[1].mod.split(mods1[1].inputs, new_splits2); - // Replace new splits with old splits - mods2[1].replace(new_splits2, splits2); - - assert(all_of(mods2[0].inputs, - [&](auto ins) { return contains(args, ins) or contains(splits1, ins); })); - assert(all_of(mods2[1].inputs, [&](auto ins) { - return contains(args, ins) or contains(splits1, ins) or contains(splits2, ins); - })); - - return {{std::move(mods1[0]), std::move(mods2[0]), std::move(mods2[1])}}; -} - -// Insert parameters into the module based on the input instructions and then -// update the map_ins to map the input to the parameter. -static void insert_params(module& m, - const std::vector& inputs, - std::unordered_map& map_ins) -{ - auto n = m.get_parameter_shapes().size(); - for(auto input : inputs) - { - if(contains(map_ins, input)) - continue; - map_ins[input] = m.add_parameter(param_name(n++), input->get_shape().as_standard()); - } -} - -void module::add_params(const std::vector& inputs, - std::unordered_map* map_ins) -{ - insert_params(*this, inputs, *map_ins); -} - -std::vector -module::fuse(const std::vector& inss, - std::unordered_map* map_ins, - module::inserter insert) -{ - std::unordered_map default_map_ins; - if(map_ins == nullptr) - map_ins = &default_map_ins; - std::vector inputs; - for(auto ins : inss) - { - for(auto input : ins->inputs()) - { - if(contains(inss, input)) - continue; - if(contains(inputs, input)) - continue; - inputs.push_back(input); - } - } - insert_params(*this, inputs, *map_ins); - return this->add_instructions(inss, map_ins, std::move(insert)); -} - -std::vector -module::fuse(const module& m, - const std::vector& inputs, - std::unordered_map* map_ins, - module::inserter insert) -{ - std::unordered_map default_map_ins; - if(map_ins == nullptr) - map_ins = &default_map_ins; - insert_params(*this, inputs, *map_ins); - auto param_map = m.get_ins_param_map(inputs, true); - for(auto&& [param, input] : param_map) - { - (*map_ins)[param] = map_ins->at(input); - } - return this->add_instructions(&m, map_ins, std::move(insert)); -} - -std::vector -module::insert_inline(instruction_ref ins, - const module& m, - const std::vector& inputs, - std::unordered_map* map_ins, - module::inserter insert) -{ - std::unordered_map default_map_ins; - if(map_ins == nullptr) - map_ins = &default_map_ins; - auto param_map = m.get_ins_param_map(inputs, true); - map_ins->insert(param_map.begin(), param_map.end()); - return this->insert_instructions(ins, &m, map_ins, std::move(insert)); -} - -void module_with_inputs::replace(instruction_ref ins, instruction_ref rep) -{ - auto it = std::find(inputs.begin(), inputs.end(), ins); - if(it == inputs.end()) - return; - assert((*it)->get_shape().lens() == rep->get_shape().lens()); - *it = rep; -} -void module_with_inputs::replace( - const std::unordered_map& map_ins) -{ - for(auto& ins : inputs) - { - if(not contains(map_ins, ins)) - continue; - assert(ins->get_shape().lens() == map_ins.at(ins)->get_shape().lens()); - ins = map_ins.at(ins); - } -} -void module_with_inputs::replace(const std::vector& keys, - const std::vector& values) -{ - for(auto& ins : inputs) - { - auto it = std::find(keys.begin(), keys.end(), ins); - if(it == keys.end()) - continue; - assert(ins->get_shape().lens() == values[it - keys.begin()]->get_shape().lens()); - ins = values[it - keys.begin()]; - } -} - -void module::debug_print() const { std::cout << *this << std::endl; } - -void module::debug_print(instruction_ref ins, - std::unordered_map& names) const -{ - if(is_end(ins, this->end())) - { - std::cout << "End instruction" << std::endl; - return; - } - if(not has_instruction(ins)) - { - std::cout << "Instruction not part of module" << std::endl; - return; - } - - names = this->print( - [&](auto x, auto ins_names) { - if(x == ins) - { - instruction::print(std::cout, x, ins_names); - std::cout << std::endl; - } - }, - names); -} - -void module::debug_print(instruction_ref ins) const -{ - std::unordered_map names; - this->debug_print(ins, names); -} - -void module::debug_print(const std::vector& inss) const -{ - for(auto ins : inss) - this->debug_print(ins); - std::cout << std::endl; -} - -std::unordered_map module::print( - const std::function&)>& print_func, - std::unordered_map names) const -{ - const bool is_root = names.empty(); - int count = 0; - for(auto ins : iterator_for(*this)) - { - std::string var_name; - if(not this->name().empty() and not is_root) - var_name = this->name() + ":"; - if(ins->name() == "@param") - { - var_name.append(any_cast(ins->get_operator()).parameter); - } - else - { - var_name.append("@" + std::to_string(count)); - } - // count every instruction so index matches loc in the printout program - count++; - names.emplace(ins, var_name); - - print_func(ins, names); - } - return names; -} - -void module::print(const std::function< - void(instruction_ref, const std::unordered_map&)>& - print_func) const -{ - this->print(print_func, {}); -} - -static std::string enclose_name(const std::string& name) -{ - return '"' + replace_string(name, "\"", "\\\"") + '"'; -} - -void module::print_graph(std::ostream& os, bool brief) const -{ - os << "digraph {" << std::endl; - os << "\trankdir=LR;" << std::endl; - this->print([&](auto ins, auto ins_names) { - std::string label; - if(brief) - label = ins->name(); - else - label = to_string(ins->get_operator()); - os << "\t" << enclose_name(ins_names.at(ins)) << "[label=" << enclose_name(label) << "]"; - os << ";" << std::endl; - if(not ins->inputs().empty()) - { - for(auto&& arg : ins->inputs()) - { - os << "\t" << enclose_name(ins_names.at(arg)) << " -> " - << enclose_name(ins_names.at(ins)); - if(not brief) - os << "[label=" << enclose_name(to_string(ins->get_shape())) << "]"; - os << ";" << std::endl; - } - } - }); - os << "}" << std::endl; -} - -static std::string cpp_var_name(const std::string& name) -{ - std::string prefix = "x_"; - if(not contains(name, "@")) - prefix = "p_"; - return to_c_id(prefix + replace_string(name, ":", "_module_")); -} - -static void print_py_op(std::ostream& os, const operation& op) -{ - auto v = op.to_value(); - os << "migraphx.op(" << enclose_name(op.name()); - - auto default_values = make_op(op.name()).to_value(); - for(auto&& x : v) - { - auto name = x.get_key(); - if(default_values[name] == x) - continue; - os << ", " << name << "=" << to_json_string(x.without_key()); - } - os << ")"; -} - -static void print_make_op(std::ostream& os, const operation& op) -{ - auto v = op.to_value(); - if(not v.empty()) - { - os << "migraphx::make_json_op(" << enclose_name(op.name()); - os << ", " << enclose_name(to_json_string(v)); - } - else - { - os << "migraphx::make_op(" << enclose_name(op.name()); - } - os << ")"; -} - -static void print_py_shape(std::ostream& os, const migraphx::shape& s) -{ - os << "migraphx.shape(type=" << to_json_string(s.type_string()) << ", lens=[" - << to_string_range(s.lens()) << "]"; - if(not s.standard()) - os << ", strides=[" << to_string_range(s.strides()) << "]"; - os << ")"; -} - -static void print_cpp_shape(std::ostream& os, const migraphx::shape& s) -{ - os << "migraphx::shape{migraphx::shape::" << s.type_string(); - os << ", {" << to_string_range(s.lens()) << "}"; - if(not s.standard()) - os << ", {" << to_string_range(s.strides()) << "}"; - os << "}"; -} - -std::unordered_map -module::print_py(std::ostream& os, - const std::string& mname, - std::unordered_map names) const -{ - // cppcheck-suppress variableScope - unsigned long seed = names.size(); - auto last = std::prev(this->end()); - names = this->print( - [&](auto ins, auto ins_names) { - std::vector input_vars; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(input_vars), - [&](auto input) { return cpp_var_name(ins_names.at(input)); }); - if(ins != last) - os << cpp_var_name(ins_names.at(ins)) << " = "; - if(ins->name() == "@literal") - { - os << mname << ".add_literal("; - if(ins->get_shape().elements() < 10) - { - os << "migraphx.create_argument("; - print_py_shape(os, ins->get_shape()); - os << ", [" << ins->get_literal() << "])"; - } - else - { - const bool use_abs = false; - // Disable abs for now - // ins->get_literal().visit([&](auto v) { - // use_abs = std::none_of(v.begin(), v.end(), [](auto x) { return x < 0; }); - // }); - if(use_abs) - os << "migraphx.abs_literal("; - os << "migraphx.generate_argument("; - print_py_shape(os, ins->get_shape()); - os << ", " << seed << ")"; - if(use_abs) - os << ")"; - seed++; - } - os << ")" << std::endl; - } - else if(ins->name() == "@param") - { - std::string name = any_cast(ins->get_operator()).parameter; - os << mname << ".add_parameter(" << enclose_name(name) << ", "; - print_py_shape(os, ins->get_shape()); - os << ")" << std::endl; - } - else if(ins->name() == "@return") - { - os << mname << ".add_return([" << join_strings(input_vars, ", ") << "])" - << std::endl; - } - else - { - assert(ins->name().front() != '@'); - os << mname << ".add_instruction("; - print_py_op(os, ins->get_operator()); - os << ", [" << join_strings(input_vars, ", ") << "]"; - os << ") # "; - print_py_shape(os, ins->get_shape()); - os << std::endl; - } - }, - names); - - return names; -} - -std::unordered_map -module::print_cpp(std::ostream& os, - const std::string& mname, - std::unordered_map names) const -{ - // cppcheck-suppress variableScope - unsigned long seed = names.size(); - auto last = std::prev(this->end()); - names = this->print( - [&](auto ins, auto ins_names) { - std::vector input_vars; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(input_vars), - [&](auto input) { return cpp_var_name(ins_names.at(input)); }); - if(ins != last) - os << "auto " << cpp_var_name(ins_names.at(ins)) << " = "; - if(ins->name() == "@literal") - { - os << mname << "->add_literal("; - bool use_abs = false; - ins->get_literal().visit([&](auto v) { - use_abs = std::none_of(v.begin(), v.end(), [](auto x) { return x < 0; }); - }); - if(use_abs) - os << "migraphx::abs("; - os << "migraphx::generate_literal("; - print_cpp_shape(os, ins->get_shape()); - os << ", " << seed << ")"; - if(use_abs) - os << ")"; - os << ");" << std::endl; - seed++; - } - else if(ins->name() == "@param") - { - std::string name = any_cast(ins->get_operator()).parameter; - os << mname << "->add_parameter(" << enclose_name(name) << ","; - print_cpp_shape(os, ins->get_shape()); - os << ");" << std::endl; - } - else if(ins->name() == "@return") - { - os << mname << "->add_return({"; - os << join_strings(input_vars, ", "); - os << "});" << std::endl; - } - else - { - assert(ins->name().front() != '@'); - os << mname << "->add_instruction("; - print_make_op(os, ins->get_operator()); - os << ", " << join_strings(input_vars, ", "); - os << ");" << std::endl; - } - }, - names); - - return names; -} - -void module::print_py(std::ostream& os) const { this->print_py(os, this->name(), {}); } - -void module::print_cpp(std::ostream& os) const { this->print_cpp(os, this->name(), {}); } - -void module::annotate(std::ostream& os, std::function a) const -{ - this->print([&](auto ins, auto ins_names) { - instruction::print(os, ins, ins_names); - a(ins); - os << std::endl; - }); -} - -std::vector module::get_sub_modules(bool shallow) const -{ - std::vector vec_modules; - for(auto ins : iterator_for(*this)) - { - const auto& mod_args = ins->module_inputs(); - vec_modules.insert(vec_modules.end(), mod_args.begin(), mod_args.end()); - if(not shallow) - { - for(const auto& smod : mod_args) - { - auto sub_mods = smod->get_sub_modules(); - vec_modules.insert(vec_modules.end(), sub_mods.begin(), sub_mods.end()); - } - } - } - - return vec_modules; -} - -module& module::sort() -{ - auto implicit_deps = calc_implicit_deps(); - fix([&](auto self, auto ins) { - this->move_instruction(ins, this->begin()); - auto ins_inputs = ins->inputs(); - if(implicit_deps.find(ins) != implicit_deps.end()) - { - auto ins_implict_inputs = implicit_deps.at(ins); - ins_inputs.insert( - ins_inputs.end(), ins_implict_inputs.begin(), ins_implict_inputs.end()); - } - for(auto child : ins_inputs) - { - if(not contains(this->impl->instructions, child)) - { - continue; - } - self(child); - } - })(std::prev(this->end())); - assert(this->validate() == this->end()); - return *this; -} - -void module::calc_implicit_deps(const module& smod, - const module& pmod, - instruction_ref ins, - ins_dep_map& deps) const -{ - const auto& ins_inputs = ins->inputs(); - for(auto ii : iterator_for(smod)) - { - const auto& ii_inputs = ii->inputs(); - for(auto iii : ii_inputs) - { - if(pmod.has_instruction(iii)) - { - if(not contains(ins_inputs, iii)) - deps[ins].insert(iii); - } - } - - const auto& mod_args = ii->module_inputs(); - for(const auto* ssmod : mod_args) - { - calc_implicit_deps(*ssmod, pmod, ins, deps); - } - } -} - -ins_dep_map module::calc_implicit_deps() const -{ - ins_dep_map mod_implicit_deps; - for(auto ins : iterator_for(*this)) - { - const auto& mod_args = ins->module_inputs(); - if(mod_args.empty()) - { - continue; - } - - for(const auto* mod : mod_args) - { - calc_implicit_deps(*mod, *this, ins, mod_implicit_deps); - } - } - - return mod_implicit_deps; -} - -void module::repeat_while_changes(std::size_t n, const std::function& f) -{ - if(n == 0) - return; - if(n == 1) - { - f(); - return; - } - auto has_changed = impl->changed.subscribe(); - for(auto i : range(n)) - { - f(); - if(not has_changed) - break; - (void)i; - } -} - -bool operator==(const module& x, const module& y) { return to_string(x) == to_string(y); } - -std::ostream& operator<<(std::ostream& os, const module& m) -{ - m.print([&](auto ins, auto ins_names) { - instruction::print(os, ins, ins_names); - os << std::endl; - }); - - return os; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/msgpack.cpp b/docker/rocm/migraphx/msgpack.cpp deleted file mode 100644 index 36c6148c5..000000000 --- a/docker/rocm/migraphx/msgpack.cpp +++ /dev/null @@ -1,256 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -// Leave an extra byte for error checking -constexpr std::size_t msgpack_size_limit = std::numeric_limits::max() - 1; - -template -std::size_t msgpack_chunk_size(const Range& r) -{ - return 1 + (r.size() - 1) / msgpack_size_limit; -} - -template -void msgpack_chunk_for_each(Iterator start, Iterator last, F f) -{ - while(std::distance(start, last) > msgpack_size_limit) - { - auto next = std::next(start, msgpack_size_limit); - f(start, next); - start = next; - } - f(start, last); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -namespace msgpack { -MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) -{ - namespace adaptor { - - template <> - struct convert - { - const msgpack::object& operator()(const msgpack::object& o, migraphx::value& v) const - { - switch(o.type) - { - case msgpack::type::NIL: { - v = nullptr; - break; - } - case msgpack::type::BOOLEAN: { - v = o.as(); - break; - } - case msgpack::type::POSITIVE_INTEGER: { - v = o.as(); - break; - } - case msgpack::type::NEGATIVE_INTEGER: { - v = o.as(); - break; - } - case msgpack::type::FLOAT32: - case msgpack::type::FLOAT64: { - v = o.as(); - break; - } - case msgpack::type::STR: { - v = o.as(); - break; - } - case msgpack::type::BIN: { - // For backwards compatibility - v = migraphx::value::binary{o.via.bin.ptr, o.via.bin.size}; - break; - } - case msgpack::type::ARRAY: { - if(o.via.array.size != 0 and o.via.array.ptr->type == msgpack::type::BIN) - { - auto bin = migraphx::value::binary{}; - std::for_each( - o.via.array.ptr, - o.via.array.ptr + o.via.array.size, - [&](const msgpack::object& so) { - bin.insert(bin.end(), so.via.bin.ptr, so.via.bin.ptr + so.via.bin.size); - }); - v = bin; - } - else - { - migraphx::value r = migraphx::value::array{}; - std::for_each( - o.via.array.ptr, - o.via.array.ptr + o.via.array.size, - [&](const msgpack::object& so) { r.push_back(so.as()); }); - v = r; - } - break; - } - case msgpack::type::MAP: { - migraphx::value r = migraphx::value::object{}; - std::for_each(o.via.map.ptr, - o.via.map.ptr + o.via.map.size, - [&](const msgpack::object_kv& p) { - r[p.key.as()] = p.val.as(); - }); - v = r; - break; - } - case msgpack::type::EXT: { - MIGRAPHX_THROW("msgpack EXT type not supported."); - } - } - return o; - } - }; - - template <> - struct pack - { - template - packer& operator()(msgpack::packer& o, - const migraphx::value::binary& x) const - { - const auto* data = reinterpret_cast(x.data()); - auto size = x.size(); - o.pack_array(migraphx::msgpack_chunk_size(x)); - migraphx::msgpack_chunk_for_each( - data, data + size, [&](const char* start, const char* last) { - o.pack_bin(last - start); - o.pack_bin_body(start, last - start); - }); - return o; - } - }; - - template <> - struct pack - { - template - void write(msgpack::packer& o, const std::nullptr_t&) const - { - o.pack_nil(); - } - template - void write(msgpack::packer& o, const T& x) const - { - o.pack(x); - } - template - void write(msgpack::packer& o, const std::vector& v) const - { - if(v.empty()) - { - o.pack_array(0); - return; - } - if(v.size() > migraphx::msgpack_size_limit) - MIGRAPHX_THROW("Size is too large for msgpack"); - if(not v.front().get_key().empty()) - { - o.pack_map(v.size()); - for(auto&& x : v) - { - o.pack(x.get_key()); - o.pack(x.without_key()); - } - } - else - { - o.pack_array(v.size()); - for(auto&& x : v) - { - o.pack(x); - } - } - } - template - packer& operator()(msgpack::packer& o, const migraphx::value& v) const - { - v.visit_value([&](auto&& x) { this->write(o, x); }); - return o; - } - }; - - } // namespace adaptor -} // MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) -} // namespace msgpack - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct vector_stream -{ - std::vector buffer{}; - vector_stream& write(const char* b, std::size_t n) - { - buffer.insert(buffer.end(), b, b + n); - return *this; - } -}; - -struct writer_stream -{ - std::function writer; - writer_stream& write(const char* b, std::size_t n) - { - writer(b, n); - return *this; - } -}; - -void to_msgpack(const value& v, std::function writer) -{ - writer_stream ws{std::move(writer)}; - msgpack::pack(ws, v); -} - -std::vector to_msgpack(const value& v) -{ - vector_stream vs; - msgpack::pack(vs, v); - return vs.buffer; -} -value from_msgpack(const char* buffer, std::size_t size) -{ - msgpack::object_handle oh = msgpack::unpack(buffer, size); - return oh.get().as(); -} -value from_msgpack(const std::vector& buffer) -{ - return from_msgpack(buffer.data(), buffer.size()); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/netron_output.cpp b/docker/rocm/migraphx/netron_output.cpp deleted file mode 100644 index 842a04de4..000000000 --- a/docker/rocm/migraphx/netron_output.cpp +++ /dev/null @@ -1,278 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace { - -// from https://onnx.ai/onnx/intro/concepts.html -int get_onnx_type(shape::type_t s_type) -{ - switch(s_type) - { - case shape::float_type: return 1; - case shape::uint8_type: return 2; - case shape::int8_type: return 3; - case shape::uint16_type: return 4; - case shape::int16_type: return 5; - case shape::int32_type: return 6; - case shape::int64_type: return 7; - case shape::bool_type: return 9; - case shape::half_type: return 10; - case shape::double_type: return 11; - case shape::uint32_type: return 12; - case shape::uint64_type: return 13; - case shape::bf16_type: return 16; - case shape::fp8e4m3fn_type: return 17; - case shape::fp8e4m3fnuz_type: return 18; - case shape::fp8e5m2_type: return 19; - case shape::fp8e5m2fnuz_type: return 20; - case shape::tuple_type: return 0; - } - MIGRAPHX_THROW("MIGraphX type " + std::to_string(s_type) + " not supported"); -} - -auto make_attribute(const migraphx::value& val) -{ - value attribute = value(std::unordered_map()); - attribute["name"] = val.get_key(); - auto val_string = val.to(); - std::string sub_str = val.get_key() + ":"; - auto find_key = val_string.find(sub_str); - if(find_key != std::string::npos) - { - val_string = val_string.substr(find_key + sub_str.length() + 1); - } - // TODO: doesn't work for some reason with Netron now - // attribute["s"] = base64_encode(val_string); - // attribute["type"] = "STRING"; - attribute["docString"] = val_string; - return attribute; -} - -/// Returns a value with the JSON structure needed for a node -auto make_onnx_json_node(instruction_ref ins, - std::unordered_map ins_uids) -{ - value node; - // TODO add support for module inputs - value input_arr = value({}); - for(instruction_ref input_ins : ins->inputs()) - { - auto name = input_ins->name(); - if(name == "@literal" or name == "@param") - { - input_arr.push_back(ins_uids.at(input_ins)); - } - // TODO make a better process for handling nodes to ignore - else if(name.find("hip::hip_allocate_memory") != std::string::npos) - { - continue; - } - else - { - input_arr.push_back(ins_uids.at(input_ins) + "->" + ins_uids.at(ins)); - } - } - value output_arr = value({}); - for(instruction_ref output_ins : ins->outputs()) - { - if(output_ins->name() == "@return") - { - output_arr.push_back(ins_uids.at(output_ins)); - } - else - { - output_arr.push_back(ins_uids.at(ins) + "->" + ins_uids.at(output_ins)); - } - } - node["input"] = input_arr; - node["output"] = output_arr; - node["name"] = ins_uids.at(ins); - node["opType"] = ins->name(); - value op_attribute_arr = value({}); - auto op_value = ins->get_operator().to_value(); - std::for_each(op_value.begin(), op_value.end(), [&](auto v) { - const std::string& attr_key = v.get_key(); - if(v.is_binary() or attr_key == "code_object") - { - return; - } - else if(attr_key == "symbol_name" or attr_key == "name") - { - node["opType"] = migraphx::from_value(v); - } - else - { - op_attribute_arr.push_back(make_attribute(v)); - } - }); - node["attribute"] = op_attribute_arr; - return node; -} - -// ONNX graph constant data called "initializer" -auto make_onnx_json_literal(instruction_ref ins, - std::unordered_map ins_uids) -{ - value lit; - lit["dims"] = ins->get_shape().lens(); - lit["dataType"] = get_onnx_type(ins->get_shape().type()); - lit["name"] = ins_uids.at(ins); - // ignoring literal data, setting to "NULL" in base64 - lit["rawData"] = "TlVMTA=="; - return lit; -} - -// TODO handle dynamic shapes -// TODO handle subshapes -auto make_onnx_json_shape(const shape& s) -{ - value ret; - value dim = value({}); - for(std::size_t len : s.lens()) - { - // cppcheck-suppress useStlAlgorithm - dim.push_back({{"dimValue", len}}); - } - ret["dim"] = dim; - return ret; -} - -// ONNX graph edges called "valueType" -auto make_onnx_json_edge(instruction_ref ins, - instruction_ref out_ins, - std::unordered_map ins_uids) -{ - value ret; - shape ins_shape = ins->get_shape(); - ret["name"] = ins_uids.at(ins) + "->" + ins_uids.at(out_ins); - value type = {{"tensorType", - {{"elemType", get_onnx_type(ins_shape.type())}, - {"shape", make_onnx_json_shape(ins_shape)}}}}; - ret["type"] = type; - return ret; -} - -auto make_onnx_json_in_out(instruction_ref ins, - std::unordered_map ins_uids) -{ - value ret; - shape ins_shape = ins->get_shape(); - ret["name"] = ins_uids.at(ins); - value type = {{"tensorType", - {{"elemType", get_onnx_type(ins_shape.type())}, - {"shape", make_onnx_json_shape(ins_shape)}}}}; - ret["type"] = type; - return ret; -} - -std::unordered_map make_ins_uids(const module& mod) -{ - std::unordered_map ret; - int count = 0; - for(auto ins : iterator_for(mod)) - { - std::string var_name; - var_name = mod.name() + ":"; - var_name.append(ins->name() + ":"); - var_name.append("@" + std::to_string(count)); - count++; - ret.emplace(ins, var_name); - } - return ret; -} - -value make_graph(const module* mod) -{ - value graph = {{"node", value({})}, - {"initializer", value({})}, - {"input", value({})}, - {"output", value({})}, - {"valueInfo", value({})}}; - auto ins_uids = make_ins_uids(*mod); - for(auto ins = mod->begin(); ins != mod->end(); ++ins) - { - const auto& name = ins->name(); - if(name == "@literal") - { - graph["initializer"].push_back(make_onnx_json_literal(ins, ins_uids)); - } - else if(name == "@param") - { - graph["input"].push_back(make_onnx_json_in_out(ins, ins_uids)); - } - else if(name == "@return") - { - graph["output"].push_back(make_onnx_json_in_out(ins, ins_uids)); - } - else if(name.find("hip::hip_allocate_memory") != std::string::npos) - { - continue; - } - else - { - graph["node"].push_back(make_onnx_json_node(ins, ins_uids)); - const auto& outputs = ins->outputs(); - for(auto out_ins : outputs) - { - if(out_ins->name() != "@return") - { - graph["valueInfo"].push_back(make_onnx_json_edge(ins, out_ins, ins_uids)); - } - } - } - } - return graph; -} - -} // namespace - -std::string make_netron_output(const program& prog) -{ - value output; - auto prog_value = prog.to_value(); - // ONNX IR version 6 - // TODO: investigate sure how this affects things - output["irVersion"] = 6; - output["producerName"] = "AMDMIGraphX"; - output["producerVersion"] = prog_value.at("migraphx_version").to(); - for(auto& mod : prog.get_modules()) - { - auto graph = make_graph(mod); - output["graph"] = graph; - } - return to_pretty_json_string(output, 4); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/normalize_attributes.cpp b/docker/rocm/migraphx/normalize_attributes.cpp deleted file mode 100644 index 1609170cf..000000000 --- a/docker/rocm/migraphx/normalize_attributes.cpp +++ /dev/null @@ -1,283 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Parameters: - * vec: the vector attribute to normalize - * axes: the operator's axes attribute if it exists, empty otherwise - * val: the normalize_axes key and options. Ex: normalize["axes"] = - * value::array{normalize_attribute::include_min}; - * input_shape: input shape passed when calling - * normalize_attributes(op&, input_shape) - * - * See normalize_attribute.hpp for explaining the options. - */ -template -auto tune_attribute(const std::vector& vec, - const std::vector& axes, - const value& val, - const shape& input_shape, - Message m) -{ - std::vector result(vec); - if(result.empty()) - { - return result; - }; - int64_t n_rank = input_shape.ndim(); - std::vector vec_attrs = val.to_vector(); - if(contains(vec_attrs, op::normalize_attribute::use_output)) - { - n_rank = n_rank + vec.size(); - } - - std::vector max_vals(vec.size(), n_rank); - - if(contains(vec_attrs, op::normalize_attribute::use_len)) - { - if(input_shape.dynamic()) - { - // return the unchanged `vec` if the dynamic_dimensions at `axes` are not fixed - if(std::any_of(axes.begin(), axes.end(), [&](auto ax) { - return not input_shape.dyn_dims().at(ax).is_fixed(); - })) - { - return vec; - } - std::transform(axes.begin(), axes.end(), max_vals.begin(), [&](auto i) { - return input_shape.dyn_dims().at(i).max; - }); - } - else - { - std::transform(axes.begin(), axes.end(), max_vals.begin(), [&](auto i) { - return input_shape.lens().at(i); - }); - } - } - - if(contains(vec_attrs, op::normalize_attribute::clip_max)) - { - if(contains(vec_attrs, op::normalize_attribute::include_max)) - { - std::transform(result.begin(), - result.end(), - max_vals.begin(), - result.begin(), - [](auto v, auto mv) { return v > mv ? mv : v; }); - } - else - { - std::transform(result.begin(), - result.end(), - max_vals.begin(), - result.begin(), - [](auto v, auto mv) { return v >= mv ? mv - 1 : v; }); - } - } - else - { - if(contains(vec_attrs, op::normalize_attribute::include_max)) - { - if(not std::equal(result.begin(), result.end(), max_vals.begin(), std::less_equal<>{})) - { - MIGRAPHX_THROW(m() + "value out of range!"); - } - } - else - { - if(not std::equal(result.begin(), result.end(), max_vals.begin(), std::less<>{})) - { - MIGRAPHX_THROW(m() + "value out of range!"); - } - } - } - - std::vector min_vals = max_vals; - std::transform(min_vals.begin(), min_vals.end(), min_vals.begin(), [](auto v) { return -v; }); - if(contains(vec_attrs, op::normalize_attribute::clip_min)) - { - if(contains(vec_attrs, op::normalize_attribute::include_min)) - { - std::transform(result.begin(), - result.end(), - min_vals.begin(), - result.begin(), - [](auto v, auto mv) { return v < mv ? mv : v; }); - } - else - { - std::transform(result.begin(), - result.end(), - min_vals.begin(), - result.begin(), - [](auto v, auto mv) { return v < mv + 1 ? mv + 1 : v; }); - } - } - else - { - if(contains(vec_attrs, op::normalize_attribute::include_min)) - { - if(not std::equal( - min_vals.begin(), min_vals.end(), result.begin(), std::less_equal<>{})) - { - MIGRAPHX_THROW(m() + "attribute out of range!"); - } - } - else - { - if(not std::equal(result.begin(), result.end(), min_vals.begin(), std::less<>{})) - { - MIGRAPHX_THROW(m() + "attribute out of range!"); - } - } - } - - std::transform( - result.begin(), result.end(), max_vals.begin(), result.begin(), [](auto v, auto mv) { - return v < 0 ? v + mv : v; - }); - - return result; -} - -auto tune_pad_attribute(const value& val) -{ - - std::vector vec_attrs = val.to_vector(); - std::vector result(vec_attrs.begin(), vec_attrs.end()); - std::copy(vec_attrs.begin(), vec_attrs.end(), std::back_inserter(result)); - - return result; -} - -/** - * Assumptions: - * Dimensions to pad start from the third dimension (index 2). - * Called by compute_shape_op() with the shape of the first input. - */ -bool normalize_attributes(operation& op, const shape& input_shape) -{ - bool tuned = false; - auto attrs = op.attributes(); - auto val = op.to_value(); - if(attrs.contains("normalize_padding")) - { - bool use_auto_padding = - (val.contains("padding_mode") and - (val.at("padding_mode").to() != migraphx::op::padding_mode_t::default_)); - if(not use_auto_padding) - { - auto padding = val.at(attrs.at("normalize_padding").to()); - auto padding_size = padding.size(); - auto padding_start = 2; - if(padding_size == 2 * (input_shape.ndim() - padding_start)) - tuned = true; - else if(padding_size != (input_shape.ndim() - padding_start)) - { - MIGRAPHX_THROW("normalize_attributes: inconsistent padding vector size "); - } - else - { - auto result = tune_pad_attribute(padding); - val["padding"] = result; - op.from_value(val); - tuned = true; - } - } - } - if(not attrs.contains("normalize_axes")) - { - return tuned; - } - - auto attr_v = attrs.at("normalize_axes").without_key(); - for(const auto& rv : attr_v) - { - const auto& key = rv.get_key(); - if(val.contains(key)) - { - auto message = [&] { return op.name() + ": " + key + ": "; }; - auto vv = val.at(key).without_key(); - if(vv.is_array()) - { - std::vector axes; - if(val.contains("axes")) - { - axes = val.at("axes").without_key().to_vector(); - } - auto vec = vv.to_vector(); - auto result = tune_attribute(vec, axes, rv.without_key(), input_shape, message); - val[key] = result; - op.from_value(val); - val = op.to_value(); - tuned = true; - } - else - { - auto num = vv.to(); - auto result = tune_attribute({num}, {num}, rv.without_key(), input_shape, message); - val[key] = result.front(); - op.from_value(val); - val = op.to_value(); - tuned = true; - } - } - else - { - MIGRAPHX_THROW("NORMALIZE_ATTR : op " + op.name() + " attribute \"" + key + - "\" not exist!"); - } - } - - return tuned; -} - -std::vector normalize_axes(const std::vector& axes, - const shape& input_shape, - const value& attr_val, - const std::string& prefix) -{ - return tune_attribute(axes, {}, attr_val, input_shape, [&] { return prefix; }); -} - -std::vector normalize_indices(const std::vector& indices, - const std::vector& axes, - const shape& input_shape, - const value& attr_val, - const std::string& prefix) -{ - return tune_attribute(indices, axes, attr_val, input_shape, [&] { return prefix; }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/normalize_ops.cpp b/docker/rocm/migraphx/normalize_ops.cpp deleted file mode 100644 index cdfa00801..000000000 --- a/docker/rocm/migraphx/normalize_ops.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void normalize_ops::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - auto inputs = ins->inputs(); - if(inputs.empty()) - continue; - - auto s = inputs[0]->get_shape(); - migraphx::operation tuned_op = ins->get_operator(); - if(normalize_attributes(tuned_op, s)) - { - m.replace_instruction(ins, tuned_op, inputs); - ins->set_normalized(); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/CMakeLists.txt b/docker/rocm/migraphx/onnx/CMakeLists.txt deleted file mode 100644 index 677abe4ef..000000000 --- a/docker/rocm/migraphx/onnx/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### -find_package(Protobuf REQUIRED) - -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS onnx.proto) -add_library(onnx-proto STATIC ${PROTO_SRCS}) -target_include_directories(onnx-proto SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${PROTOBUF_INCLUDE_DIR}) -if(MSVC) - target_compile_options(onnx-proto PRIVATE /w) -else() - target_compile_options(onnx-proto PRIVATE -w) -endif() -target_link_libraries(onnx-proto PRIVATE ${PROTOBUF_LIBRARY}) -set_target_properties(onnx-proto PROPERTIES POSITION_INDEPENDENT_CODE On) - -file(GLOB ONNX_SRCS CONFIGURE_DEPENDS *.cpp) -add_library(migraphx_onnx ${ONNX_SRCS}) -target_include_directories(migraphx_onnx PRIVATE include) -set_target_properties(migraphx_onnx PROPERTIES EXPORT_NAME onnx) -migraphx_generate_export_header(migraphx_onnx) -rocm_set_soversion(migraphx_onnx ${MIGRAPHX_SO_VERSION}) -rocm_clang_tidy_check(migraphx_onnx) -target_link_libraries(migraphx_onnx PRIVATE onnx-proto) -if(NOT WIN32) - target_link_libraries(migraphx_onnx PRIVATE "-Wl,--exclude-libs,ALL") -endif() -target_link_libraries(migraphx_onnx PUBLIC migraphx) - -rocm_install_targets( - PRIVATE - TARGETS migraphx_onnx -) diff --git a/docker/rocm/migraphx/onnx/broadcast_qdq.cpp b/docker/rocm/migraphx/onnx/broadcast_qdq.cpp deleted file mode 100644 index dbe6878a4..000000000 --- a/docker/rocm/migraphx/onnx/broadcast_qdq.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// This method is to prep for quantizelinear or dequantizelinear operation for -// either the broadcasting of weight-scale or zero-points of qlinearadd operator -// outputs: operator op (inputs x, broadcasted: scale (float) & zero_pt (8-bit)) -instruction_ref bcast_qdq_instr(const std::string& op_name, - instruction_ref x_in, - instruction_ref arg_fscale, - instruction_ref arg_z_pt, - const onnx_parser::node_info& info) -{ - auto in_lens = x_in->get_shape().lens(); - - // prep 1: broadcast scale. it can come as a scalar or a 1-D tensor. - instruction_ref bcast_scale; - if(arg_fscale->get_shape().elements() > 1) - bcast_scale = info.add_instruction( - migraphx::make_op("broadcast", {{"axis", 0}, {"out_lens", in_lens}}), arg_fscale); - else - bcast_scale = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", in_lens}}), arg_fscale); - - // prep 2: broadcast zero point. it can come as a scalar or a 1-D tensor. - instruction_ref bcast_zero_pt; - if(arg_z_pt->get_shape().elements() > 1) - bcast_zero_pt = info.add_instruction( - migraphx::make_op("broadcast", {{"axis", 0}, {"out_lens", in_lens}}), arg_z_pt); - else - bcast_zero_pt = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", in_lens}}), arg_z_pt); - - // op_name is either quantizelinear or dequantizelinear: - return info.add_instruction(migraphx::make_op(op_name), x_in, bcast_scale, bcast_zero_pt); -} - -// Multibroadcast a scaler.. -instruction_ref bcast_scalar_instr(const migraphx::shape& shape_out, - instruction_ref arg_in, - const onnx_parser::node_info& info) -{ - return info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", shape_out.lens()}}), arg_in); -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/checks.cpp b/docker/rocm/migraphx/onnx/checks.cpp deleted file mode 100644 index 70c382f05..000000000 --- a/docker/rocm/migraphx/onnx/checks.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void check_arg_empty(const argument& arg, const std::string& msg) -{ - if(arg.empty()) - { - MIGRAPHX_THROW(msg); - } -} - -void check_attr_sizes(size_t kdims, size_t attr_size, const std::string& error_msg) -{ - if(kdims != attr_size) - { - MIGRAPHX_THROW(error_msg + " k-dims: " + std::to_string(kdims) + - " attribute size: " + std::to_string(attr_size)); - } -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/conv.cpp b/docker/rocm/migraphx/onnx/conv.cpp deleted file mode 100644 index c37a2e1d8..000000000 --- a/docker/rocm/migraphx/onnx/conv.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void recalc_conv_attributes(value& v, size_t kdims) -{ - if(v["padding"].size() != kdims and v["padding"].size() != kdims * 2) - { - v["padding"].resize(kdims); - std::fill_n(v["padding"].begin(), kdims, 0); - } - if(v["stride"].size() != kdims) - { - v["stride"].resize(kdims); - std::fill_n(v["stride"].begin(), kdims, 1); - } - if(v["dilation"].size() != kdims) - { - v["dilation"].resize(kdims); - std::fill_n(v["dilation"].begin(), kdims, 1); - } -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/broadcast_qdq.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/broadcast_qdq.hpp deleted file mode 100644 index 04432b01d..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/broadcast_qdq.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_BROADCAST_QDQ_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_BROADCAST_QDQ_HPP - -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// This method is to prep for quantizelinear or dequantizelinear operation for -// either the broadcasting of weight-scale or zero-points of qlinearadd operator -// outputs: operator op (inputs x, broadcasted: scale (float) & zero_pt (8-bit)) -instruction_ref bcast_qdq_instr(const std::string& op_name, - instruction_ref x_in, - instruction_ref arg_fscale, - instruction_ref arg_z_pt, - const onnx_parser::node_info& info); - -// Multibroadcast a scaler.. -instruction_ref bcast_scalar_instr(const migraphx::shape& shape_out, - instruction_ref arg_in, - const onnx_parser::node_info& info); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/checks.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/checks.hpp deleted file mode 100644 index b018507be..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/checks.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_CHECKS_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_CHECKS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void check_arg_empty(const argument& arg, const std::string& msg); -void check_attr_sizes(size_t kdims, size_t attr_size, const std::string& error_msg); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/conv.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/conv.hpp deleted file mode 100644 index 127d0638a..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/conv.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_CONV_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_CONV_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void recalc_conv_attributes(value& v, size_t kdims); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/map_activation_functions.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/map_activation_functions.hpp deleted file mode 100644 index 5461598c2..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/map_activation_functions.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_MAP_ACTIVATION_FUNCTIONS_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_MAP_ACTIVATION_FUNCTIONS_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -const std::unordered_map& map_activation_functions(); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/onnx_parser.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/onnx_parser.hpp deleted file mode 100644 index 493ad845a..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/onnx_parser.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_PARSER_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_PARSER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -namespace onnx = onnx_for_migraphx; - -struct onnx_parser -{ - std::string filename; - fs::path path; - std::string external_data_path; - using attribute_map = std::unordered_map; - struct node_info - { - attribute_map attributes{}; - std::size_t num_outputs = 1; - std::string name = ""; - module* mod = nullptr; - instruction_ref make_contiguous(instruction_ref ins) const; - instruction_ref add_bias(const std::vector& args, - instruction_ref curr_ins, - uint64_t axis) const; - - instruction_ref add_broadcastable_binary_op(const std::string& op_name, - instruction_ref arg0, - instruction_ref arg1) const; - - instruction_ref add_common_op(const std::string& op_name, - std::vector inputs) const; - - template - instruction_ref add_common_op(const std::string& op_name, Ts... xs) const - { - return add_common_op(op_name, {xs...}); - } - - instruction_ref add_instruction(const operation& op, - const std::vector& args) const; - - instruction_ref add_instruction(const operation& op, - const std::vector& args, - const std::vector& mods) const; - - template - instruction_ref add_instruction(const operation& op, Ts... xs) const - { - return add_instruction(op, {xs...}); - } - instruction_ref add_literal(literal l) const; - template - instruction_ref add_literal(Ts&&... xs) const - { - return add_literal(literal{std::forward(xs)...}); - } - }; - using node_map = std::unordered_map; - using op_func = std::function( - onnx_parser&, const node_info&, std::vector)>; - node_map nodes; - std::unordered_map instructions; - program prog = program(); - shape::dynamic_dimension default_dyn_dim_value = {1, 1}; - std::unordered_map> map_input_dims; - std::unordered_map dim_params; - std::unordered_map> map_dyn_input_dims; - bool use_dyn_output = false; - bool skip_unknown_operators = false; - int64_t max_loop_iterations = 10; - int64_t limit_max_iterations = std::numeric_limits::max(); - int64_t opset_version = 13; - - std::unordered_map ops; - - onnx_parser(); - operation load(const std::string& name, const node_info& info) const; - - void parse_undefined(module* mod, const std::string& name); - - static int64_t get_opset_version(const onnx::ModelProto& model); - - void parse_from(std::istream& is, std::string name = ""); - void parse_from(const void* data, std::size_t size); - std::vector - parse_graph(module* mod, const onnx::GraphProto& graph, bool inlining = false); - literal parse_value(const onnx::AttributeProto& attr) const; - literal parse_tensor(const onnx::TensorProto& t) const; - shape parse_type(const onnx::TypeProto& t) const; - shape parse_type(const onnx::TypeProto& t, const std::vector& input_dims) const; -}; - -shape::type_t get_type(int dtype); -bool is_type_float(shape::type_t dtype); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/op_parser.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/op_parser.hpp deleted file mode 100644 index 2d37a5f42..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/op_parser.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_REGISTER_OP_PARSER_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_REGISTER_OP_PARSER_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct op_desc -{ - std::string onnx_name = ""; - std::string op_name = ""; -}; - -void register_op_parser(const std::string& name, onnx_parser::op_func f); -onnx_parser::op_func get_op_parser(const std::string& name); -std::vector get_op_parsers(); - -inline std::vector implicit_multi_op(std::vector inss) -{ - return inss; -} - -inline std::vector implicit_multi_op(instruction_ref ins) { return {ins}; } - -template -void register_op_parser() -{ - T parser; - for(auto&& opd : parser.operators()) - register_op_parser(opd.onnx_name, [opd, parser](auto&&... xs) { - return implicit_multi_op(parser.parse(opd, xs...)); - }); -} - -struct register_op_parser_action -{ - template - static void apply() - { - register_op_parser(); - } -}; - -template -using op_parser = auto_register; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/padding.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/padding.hpp deleted file mode 100644 index 132efbb2a..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/padding.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_PADDING_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_PADDING_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -bool is_asym_padding(const std::vector& padding); - -void cal_auto_padding_size(onnx_parser::node_info info, - value& v, - const std::vector& k_lens, - const std::vector& dilation, - const std::vector& in_lens, - std::vector& paddings); - -void check_padding_mode(const onnx_parser::node_info& info, const std::string& onnx_name); - -void tune_padding_size(const value& v, - std::vector& padding, - int count_include_pad, - std::vector& s_start); - -void check_asym_padding(const onnx_parser::node_info& info, - instruction_ref& ins, - const std::vector& padding, - value& v, - int count_include_pad = 0, - float pad_val = 0); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/pooling.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/pooling.hpp deleted file mode 100644 index e21b96237..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/pooling.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_POOLING_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_POOLING_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -value handle_pooling_values(const op_desc& opd, - onnx_parser::node_info info, - const shape& in_shape, - value values); - -instruction_ref add_pooling_op(const op_desc& opd, onnx_parser::node_info info, instruction_ref l0); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/include/migraphx/onnx/quantize_dequantize_linear.hpp b/docker/rocm/migraphx/onnx/include/migraphx/onnx/quantize_dequantize_linear.hpp deleted file mode 100644 index 56e1eb4e0..000000000 --- a/docker/rocm/migraphx/onnx/include/migraphx/onnx/quantize_dequantize_linear.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_QUANTIZE_DEQUANTIZE_LINEAR_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_ONNX_QUANTIZE_DEQUANTIZE_LINEAR_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -std::vector -transform_quantize_dequantize_linear_inputs(const onnx_parser::node_info& info, - const std::string& onnx_name, - int block_size, - int axis, - std::vector args); - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/onnx/map_activation_functions.cpp b/docker/rocm/migraphx/onnx/map_activation_functions.cpp deleted file mode 100644 index 59a987157..000000000 --- a/docker/rocm/migraphx/onnx/map_activation_functions.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -const std::unordered_map& map_activation_functions() -{ - static const std::unordered_map m = { - {"tanh", make_op("tanh")}, - {"relu", make_op("relu")}, - {"sigmoid", make_op("sigmoid")}, - {"leakyrelu", make_op("leaky_relu")}, - {"elu", make_op("elu")}}; - return m; -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/onnx.cpp b/docker/rocm/migraphx/onnx/onnx.cpp deleted file mode 100644 index a2c3db80b..000000000 --- a/docker/rocm/migraphx/onnx/onnx.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -program parse_onnx_from(const onnx_options& options, Ts&&... xs) -{ - onnx::onnx_parser parser; - parser.external_data_path = options.external_data_path; - parser.map_input_dims = options.map_input_dims; - parser.dim_params = options.dim_params; - parser.map_dyn_input_dims = options.map_dyn_input_dims; - auto dim_val = options.default_dim_value; - if(dim_val != 0) - { - if(options.default_dyn_dim_value != shape::dynamic_dimension{1, 1}) - { - MIGRAPHX_THROW("PARSE_ONNX_FROM: both default_dim_value and default_dyn_dim_value" - "set to non-default value"); - } - else - { - parser.default_dyn_dim_value = {dim_val, dim_val}; - } - } - else - { - parser.default_dyn_dim_value = options.default_dyn_dim_value; - } - if(not options.map_input_dims.empty() and not options.map_dyn_input_dims.empty()) - { - MIGRAPHX_THROW("PARSE_ONNX_FROM: both map_input_dims and map_dyn_input_dims non-empty, only" - "one should be used"); - } - parser.skip_unknown_operators = options.skip_unknown_operators; - parser.max_loop_iterations = options.max_loop_iterations; - parser.limit_max_iterations = options.limit_max_iterations; - parser.use_dyn_output = options.use_dyn_output; - - if(options.print_program_on_error) - { - // Log the program when it can't be parsed - try - { - parser.parse_from(std::forward(xs)...); - } - catch(...) - { - std::cerr << parser.prog << std::endl; - throw; - } - } - else - { - parser.parse_from(std::forward(xs)...); - } - - return std::move(parser.prog); -} - -program parse_onnx(const std::string& name, const onnx_options& options) -{ - std::fstream input(name.c_str(), std::ios::in | std::ios::binary); - return parse_onnx_from(options, input, name); -} - -program parse_onnx_buffer(const std::string& buffer, const onnx_options& options) -{ - return parse_onnx_from(options, buffer.data(), buffer.size()); -} - -program parse_onnx_buffer(const void* data, std::size_t size, const onnx_options& options) -{ - return parse_onnx_from(options, data, size); -} - -std::vector get_onnx_operators() { return onnx::get_op_parsers(); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/onnx.proto b/docker/rocm/migraphx/onnx/onnx.proto deleted file mode 100644 index 4f3e558fb..000000000 --- a/docker/rocm/migraphx/onnx/onnx.proto +++ /dev/null @@ -1,872 +0,0 @@ -// -// WARNING: This file is automatically generated! Please edit onnx.in.proto. -// - - -// SPDX-License-Identifier: Apache-2.0 - - -syntax = "proto2"; - -package onnx_for_migraphx; - -// Overview -// -// ONNX is an open specification that is comprised of the following components: -// -// 1) A definition of an extensible computation graph model. -// 2) Definitions of standard data types. -// 3) Definitions of built-in operators. -// -// This document describes the syntax of models and their computation graphs, -// as well as the standard data types. Together, they are referred to as the ONNX -// Intermediate Representation, or 'IR' for short. -// -// The normative semantic specification of the ONNX IR is found in docs/IR.md. -// Definitions of the built-in neural network operators may be found in docs/Operators.md. - -// Notes -// -// Protobuf compatibility -// -// To simplify framework compatibility, ONNX is defined using the subset of protobuf -// that is compatible with both protobuf v2 and v3. This means that we do not use any -// protobuf features that are only available in one of the two versions. -// -// Here are the most notable contortions we have to carry out to work around -// these limitations: -// -// - No 'map' (added protobuf 3.0). We instead represent mappings as lists -// of key-value pairs, where order does not matter and duplicates -// are not allowed. - - -// Versioning -// -// ONNX versioning is specified in docs/IR.md and elaborated on in docs/Versioning.md -// -// To be compatible with both proto2 and proto3, we will use a version number -// that is not defined by the default value but an explicit enum number. -enum Version { - // proto3 requires the first enum value to be zero. - // We add this just to appease the compiler. - _START_VERSION = 0; - // The version field is always serialized and we will use it to store the - // version that the graph is generated from. This helps us set up version - // control. - // For the IR, we are using simple numbers starting with 0x00000001, - // which was the version we published on Oct 10, 2017. - IR_VERSION_2017_10_10 = 0x0000000000000001; - - // IR_VERSION 2 published on Oct 30, 2017 - // - Added type discriminator to AttributeProto to support proto3 users - IR_VERSION_2017_10_30 = 0x0000000000000002; - - // IR VERSION 3 published on Nov 3, 2017 - // - For operator versioning: - // - Added new message OperatorSetIdProto - // - Added opset_import in ModelProto - // - For vendor extensions, added domain in NodeProto - IR_VERSION_2017_11_3 = 0x0000000000000003; - - // IR VERSION 4 published on Jan 22, 2019 - // - Relax constraint that initializers should be a subset of graph inputs - // - Add type BFLOAT16 - IR_VERSION_2019_1_22 = 0x0000000000000004; - - // IR VERSION 5 published on March 18, 2019 - // - Add message TensorAnnotation. - // - Add quantization annotation in GraphProto to map tensor with its scale and zero point quantization parameters. - IR_VERSION_2019_3_18 = 0x0000000000000005; - - // IR VERSION 6 published on Sep 19, 2019 - // - Add support for sparse tensor constants stored in model. - // - Add message SparseTensorProto - // - Add sparse initializers - IR_VERSION_2019_9_19 = 0x0000000000000006; - - // IR VERSION 7 published on May 8, 2020 - // - Add support to allow function body graph to rely on multiple external opreator sets. - // - Add a list to promote inference graph's initializers to global and - // mutable variables. Global variables are visible in all graphs of the - // stored models. - // - Add message TrainingInfoProto to store initialization - // method and training algorithm. The execution of TrainingInfoProto - // can modify the values of mutable variables. - // - Implicitly add inference graph into each TrainingInfoProto's algorithm. - IR_VERSION_2020_5_8 = 0x0000000000000007; - - // IR VERSION 8 published on July 30, 2021 - // Introduce TypeProto.SparseTensor - // Introduce TypeProto.Optional - // Added a list of FunctionProtos local to the model - // Deprecated since_version and operator status from FunctionProto - IR_VERSION_2021_7_30 = 0x0000000000000008; - - // IR VERSION 9 published on May 5, 2023 - // Added AttributeProto to FunctionProto so that default attribute values can be set. - // Added FLOAT8E4M3FN, FLOAT8E4M3FNUZ, FLOAT8E5M2, FLOAT8E5M2FNUZ. - IR_VERSION_2023_5_5 = 0x0000000000000009; - - // IR VERSION 10 published on TBD - // Added UINT4, INT4. - IR_VERSION = 0x000000000000000A; -} - -// Attributes -// -// A named attribute containing either singular float, integer, string, graph, -// and tensor values, or repeated float, integer, string, graph, and tensor values. -// An AttributeProto MUST contain the name field, and *only one* of the -// following content fields, effectively enforcing a C/C++ union equivalent. -message AttributeProto { - reserved 12, 16 to 19; - reserved "v"; - - // Note: this enum is structurally identical to the OpSchema::AttrType - // enum defined in schema.h. If you rev one, you likely need to rev the other. - enum AttributeType { - UNDEFINED = 0; - FLOAT = 1; - INT = 2; - STRING = 3; - TENSOR = 4; - GRAPH = 5; - SPARSE_TENSOR = 11; - TYPE_PROTO = 13; - - FLOATS = 6; - INTS = 7; - STRINGS = 8; - TENSORS = 9; - GRAPHS = 10; - SPARSE_TENSORS = 12; - TYPE_PROTOS = 14; - } - - // The name field MUST be present for this version of the IR. - optional string name = 1; // namespace Attribute - - // if ref_attr_name is not empty, ref_attr_name is the attribute name in parent function. - // In this case, this AttributeProto does not contain data, and it's a reference of attribute - // in parent scope. - // NOTE: This should ONLY be used in function (sub-graph). It's invalid to be used in main graph. - optional string ref_attr_name = 21; - - // A human-readable documentation for this attribute. Markdown is allowed. - optional string doc_string = 13; - - // The type field MUST be present for this version of the IR. - // For 0.0.1 versions of the IR, this field was not defined, and - // implementations needed to use has_field heuristics to determine - // which value field was in use. For IR_VERSION 0.0.2 or later, this - // field MUST be set and match the f|i|s|t|... field in use. This - // change was made to accommodate proto3 implementations. - optional AttributeType type = 20; // discriminator that indicates which field below is in use - - // Exactly ONE of the following fields must be present for this version of the IR - optional float f = 2; // float - optional int64 i = 3; // int - optional bytes s = 4; // UTF-8 string - optional TensorProto t = 5; // tensor value - optional GraphProto g = 6; // graph - optional SparseTensorProto sparse_tensor = 22; // sparse tensor value - // Do not use field below, it's deprecated. - // optional ValueProto v = 12; // value - subsumes everything but graph - optional TypeProto tp = 14; // type proto - - repeated float floats = 7; // list of floats - repeated int64 ints = 8; // list of ints - repeated bytes strings = 9; // list of UTF-8 strings - repeated TensorProto tensors = 10; // list of tensors - repeated GraphProto graphs = 11; // list of graph - repeated SparseTensorProto sparse_tensors = 23; // list of sparse tensors - repeated TypeProto type_protos = 15;// list of type protos -} - -// Defines information on value, including the name, the type, and -// the shape of the value. -message ValueInfoProto { - // This field MUST be present in this version of the IR. - optional string name = 1; // namespace Value - // This field MUST be present in this version of the IR for - // inputs and outputs of the top-level graph. - optional TypeProto type = 2; - // A human-readable documentation for this value. Markdown is allowed. - optional string doc_string = 3; - // Named metadata values; keys should be distinct. - repeated StringStringEntryProto metadata_props = 4; -} - -// Nodes -// -// Computation graphs are made up of a DAG of nodes, which represent what is -// commonly called a "layer" or "pipeline stage" in machine learning frameworks. -// -// For example, it can be a node of type "Conv" that takes in an image, a filter -// tensor and a bias tensor, and produces the convolved output. -message NodeProto { - repeated string input = 1; // namespace Value - repeated string output = 2; // namespace Value - - // An optional identifier for this node in a graph. - // This field MAY be absent in this version of the IR. - optional string name = 3; // namespace Node - - // The symbolic identifier of the Operator to execute. - optional string op_type = 4; // namespace Operator - // The domain of the OperatorSet that specifies the operator named by op_type. - optional string domain = 7; // namespace Domain - // Overload identifier, used only to map this to a model-local function. - optional string overload = 8; - - // Additional named attributes. - repeated AttributeProto attribute = 5; - - // A human-readable documentation for this node. Markdown is allowed. - optional string doc_string = 6; - - // Named metadata values; keys should be distinct. - repeated StringStringEntryProto metadata_props = 9; -} - -// Training information -// TrainingInfoProto stores information for training a model. -// In particular, this defines two functionalities: an initialization-step -// and a training-algorithm-step. Initialization resets the model -// back to its original state as if no training has been performed. -// Training algorithm improves the model based on input data. -// -// The semantics of the initialization-step is that the initializers -// in ModelProto.graph and in TrainingInfoProto.algorithm are first -// initialized as specified by the initializers in the graph, and then -// updated by the "initialization_binding" in every instance in -// ModelProto.training_info. -// -// The field "algorithm" defines a computation graph which represents a -// training algorithm's step. After the execution of a -// TrainingInfoProto.algorithm, the initializers specified by "update_binding" -// may be immediately updated. If the targeted training algorithm contains -// consecutive update steps (such as block coordinate descent methods), -// the user needs to create a TrainingInfoProto for each step. -message TrainingInfoProto { - // This field describes a graph to compute the initial tensors - // upon starting the training process. Initialization graph has no input - // and can have multiple outputs. Usually, trainable tensors in neural - // networks are randomly initialized. To achieve that, for each tensor, - // the user can put a random number operator such as RandomNormal or - // RandomUniform in TrainingInfoProto.initialization.node and assign its - // random output to the specific tensor using "initialization_binding". - // This graph can also set the initializers in "algorithm" in the same - // TrainingInfoProto; a use case is resetting the number of training - // iteration to zero. - // - // By default, this field is an empty graph and its evaluation does not - // produce any output. Thus, no initializer would be changed by default. - optional GraphProto initialization = 1; - - // This field represents a training algorithm step. Given required inputs, - // it computes outputs to update initializers in its own or inference graph's - // initializer lists. In general, this field contains loss node, gradient node, - // optimizer node, increment of iteration count. - // - // An execution of the training algorithm step is performed by executing the - // graph obtained by combining the inference graph (namely "ModelProto.graph") - // and the "algorithm" graph. That is, the actual - // input/initializer/output/node/value_info/sparse_initializer list of - // the training graph is the concatenation of - // "ModelProto.graph.input/initializer/output/node/value_info/sparse_initializer" - // and "algorithm.input/initializer/output/node/value_info/sparse_initializer" - // in that order. This combined graph must satisfy the normal ONNX conditions. - // Now, let's provide a visualization of graph combination for clarity. - // Let the inference graph (i.e., "ModelProto.graph") be - // tensor_a, tensor_b -> MatMul -> tensor_c -> Sigmoid -> tensor_d - // and the "algorithm" graph be - // tensor_d -> Add -> tensor_e - // The combination process results - // tensor_a, tensor_b -> MatMul -> tensor_c -> Sigmoid -> tensor_d -> Add -> tensor_e - // - // Notice that an input of a node in the "algorithm" graph may reference the - // output of a node in the inference graph (but not the other way round). Also, inference - // node cannot reference inputs of "algorithm". With these restrictions, inference graph - // can always be run independently without training information. - // - // By default, this field is an empty graph and its evaluation does not - // produce any output. Evaluating the default training step never - // update any initializers. - optional GraphProto algorithm = 2; - - // This field specifies the bindings from the outputs of "initialization" to - // some initializers in "ModelProto.graph.initializer" and - // the "algorithm.initializer" in the same TrainingInfoProto. - // See "update_binding" below for details. - // - // By default, this field is empty and no initializer would be changed - // by the execution of "initialization". - repeated StringStringEntryProto initialization_binding = 3; - - // Gradient-based training is usually an iterative procedure. In one gradient - // descent iteration, we apply - // - // x = x - r * g - // - // where "x" is the optimized tensor, "r" stands for learning rate, and "g" is - // gradient of "x" with respect to a chosen loss. To avoid adding assignments - // into the training graph, we split the update equation into - // - // y = x - r * g - // x = y - // - // The user needs to save "y = x - r * g" into TrainingInfoProto.algorithm. To - // tell that "y" should be assigned to "x", the field "update_binding" may - // contain a key-value pair of strings, "x" (key of StringStringEntryProto) - // and "y" (value of StringStringEntryProto). - // For a neural network with multiple trainable (mutable) tensors, there can - // be multiple key-value pairs in "update_binding". - // - // The initializers appears as keys in "update_binding" are considered - // mutable variables. This implies some behaviors - // as described below. - // - // 1. We have only unique keys in all "update_binding"s so that two - // variables may not have the same name. This ensures that one - // variable is assigned up to once. - // 2. The keys must appear in names of "ModelProto.graph.initializer" or - // "TrainingInfoProto.algorithm.initializer". - // 3. The values must be output names of "algorithm" or "ModelProto.graph.output". - // 4. Mutable variables are initialized to the value specified by the - // corresponding initializer, and then potentially updated by - // "initializer_binding"s and "update_binding"s in "TrainingInfoProto"s. - // - // This field usually contains names of trainable tensors - // (in ModelProto.graph), optimizer states such as momentums in advanced - // stochastic gradient methods (in TrainingInfoProto.graph), - // and number of training iterations (in TrainingInfoProto.graph). - // - // By default, this field is empty and no initializer would be changed - // by the execution of "algorithm". - repeated StringStringEntryProto update_binding = 4; -} - -// Models -// -// ModelProto is a top-level file/container format for bundling a ML model and -// associating its computation graph with metadata. -// -// The semantics of the model are described by the associated GraphProto's. -message ModelProto { - // The version of the IR this model targets. See Version enum above. - // This field MUST be present. - optional int64 ir_version = 1; - - // The OperatorSets this model relies on. - // All ModelProtos MUST have at least one entry that - // specifies which version of the ONNX OperatorSet is - // being imported. - // - // All nodes in the ModelProto's graph will bind against the operator - // with the same-domain/same-op_type operator with the HIGHEST version - // in the referenced operator sets. - repeated OperatorSetIdProto opset_import = 8; - - // The name of the framework or tool used to generate this model. - // This field SHOULD be present to indicate which implementation/tool/framework - // emitted the model. - optional string producer_name = 2; - - // The version of the framework or tool used to generate this model. - // This field SHOULD be present to indicate which implementation/tool/framework - // emitted the model. - optional string producer_version = 3; - - // Domain name of the model. - // We use reverse domain names as name space indicators. For example: - // `com.facebook.fair` or `com.microsoft.cognitiveservices` - // - // Together with `model_version` and GraphProto.name, this forms the unique identity of - // the graph. - optional string domain = 4; - - // The version of the graph encoded. See Version enum below. - optional int64 model_version = 5; - - // A human-readable documentation for this model. Markdown is allowed. - optional string doc_string = 6; - - // The parameterized graph that is evaluated to execute the model. - optional GraphProto graph = 7; - - // Named metadata values; keys should be distinct. - repeated StringStringEntryProto metadata_props = 14; - - // Training-specific information. Sequentially executing all stored - // `TrainingInfoProto.algorithm`s and assigning their outputs following - // the corresponding `TrainingInfoProto.update_binding`s is one training - // iteration. Similarly, to initialize the model - // (as if training hasn't happened), the user should sequentially execute - // all stored `TrainingInfoProto.initialization`s and assigns their outputs - // using `TrainingInfoProto.initialization_binding`s. - // - // If this field is empty, the training behavior of the model is undefined. - repeated TrainingInfoProto training_info = 20; - - // A list of function protos local to the model. - // - // The (domain, name, overload) tuple must be unique across the function protos in this list. - // In case of any conflicts the behavior (whether the model local functions are given higher priority, - // or standard operator sets are given higher priotity or this is treated as error) is defined by - // the runtimes. - // - // The operator sets imported by FunctionProto should be compatible with the ones - // imported by ModelProto and other model local FunctionProtos. - // Example, if same operator set say 'A' is imported by a FunctionProto and ModelProto - // or by 2 FunctionProtos then versions for the operator set may be different but, - // the operator schema returned for op_type, domain, version combination - // for both the versions should be same for every node in the function body. - // - // One FunctionProto can reference other FunctionProto in the model, however, recursive reference - // is not allowed. - repeated FunctionProto functions = 25; -}; - -// StringStringEntryProto follows the pattern for cross-proto-version maps. -// See https://developers.google.com/protocol-buffers/docs/proto3#maps -message StringStringEntryProto { - optional string key = 1; - optional string value = 2; -}; - -message TensorAnnotation { - optional string tensor_name = 1; - // pairs to annotate tensor specified by above. - // The keys used in the mapping below must be pre-defined in ONNX spec. - // For example, for 8-bit linear quantization case, 'SCALE_TENSOR', 'ZERO_POINT_TENSOR' will be pre-defined as - // quantization parameter keys. - repeated StringStringEntryProto quant_parameter_tensor_names = 2; -} - - - -// Graphs -// -// A graph defines the computational logic of a model and is comprised of a parameterized -// list of nodes that form a directed acyclic graph based on their inputs and outputs. -// This is the equivalent of the "network" or "graph" in many deep learning -// frameworks. -message GraphProto { - // The nodes in the graph, sorted topologically. - repeated NodeProto node = 1; - - // The name of the graph. - optional string name = 2; // namespace Graph - - // A list of named tensor values, used to specify constant inputs of the graph. - // Each initializer (both TensorProto as well SparseTensorProto) MUST have a name. - // The name MUST be unique across both initializer and sparse_initializer, - // but the name MAY also appear in the input list. - repeated TensorProto initializer = 5; - - // Initializers (see above) stored in sparse format. - repeated SparseTensorProto sparse_initializer = 15; - - // A human-readable documentation for this graph. Markdown is allowed. - optional string doc_string = 10; - - // The inputs and outputs of the graph. - repeated ValueInfoProto input = 11; - repeated ValueInfoProto output = 12; - - // Information for the values in the graph. The ValueInfoProto.name's - // must be distinct. It is optional for a value to appear in value_info list. - repeated ValueInfoProto value_info = 13; - - // This field carries information to indicate the mapping among a tensor and its - // quantization parameter tensors. For example: - // For tensor 'a', it may have {'SCALE_TENSOR', 'a_scale'} and {'ZERO_POINT_TENSOR', 'a_zero_point'} annotated, - // which means, tensor 'a_scale' and tensor 'a_zero_point' are scale and zero point of tensor 'a' in the model. - repeated TensorAnnotation quantization_annotation = 14; - - // Named metadata values; keys should be distinct. - repeated StringStringEntryProto metadata_props = 16; - - reserved 3, 4, 6 to 9; - reserved "ir_version", "producer_version", "producer_tag", "domain"; -} - -// Tensors -// -// A serialized tensor value. -message TensorProto { - enum DataType { - UNDEFINED = 0; - // Basic types. - FLOAT = 1; // float - UINT8 = 2; // uint8_t - INT8 = 3; // int8_t - UINT16 = 4; // uint16_t - INT16 = 5; // int16_t - INT32 = 6; // int32_t - INT64 = 7; // int64_t - STRING = 8; // string - BOOL = 9; // bool - - // IEEE754 half-precision floating-point format (16 bits wide). - // This format has 1 sign bit, 5 exponent bits, and 10 mantissa bits. - FLOAT16 = 10; - - DOUBLE = 11; - UINT32 = 12; - UINT64 = 13; - COMPLEX64 = 14; // complex with float32 real and imaginary components - COMPLEX128 = 15; // complex with float64 real and imaginary components - - // Non-IEEE floating-point format based on IEEE754 single-precision - // floating-point number truncated to 16 bits. - // This format has 1 sign bit, 8 exponent bits, and 7 mantissa bits. - BFLOAT16 = 16; - - // Non-IEEE floating-point format based on papers - // FP8 Formats for Deep Learning, https://arxiv.org/abs/2209.05433, - // 8-bit Numerical Formats For Deep Neural Networks, https://arxiv.org/pdf/2206.02915.pdf. - // Operators supported FP8 are Cast, CastLike, QuantizeLinear, DequantizeLinear. - // The computation usually happens inside a block quantize / dequantize - // fused by the runtime. - FLOAT8E4M3FN = 17; // float 8, mostly used for coefficients, supports nan, not inf - FLOAT8E4M3FNUZ = 18; // float 8, mostly used for coefficients, supports nan, not inf, no negative zero - FLOAT8E5M2 = 19; // follows IEEE 754, supports nan, inf, mostly used for gradients - FLOAT8E5M2FNUZ = 20; // follows IEEE 754, supports nan, not inf, mostly used for gradients, no negative zero - - // 4-bit data-types - UINT4 = 21; // Unsigned integer in range [0, 15] - INT4 = 22; // Signed integer in range [-8, 7], using two's-complement representation - - // Future extensions go here. - } - - // The shape of the tensor. - repeated int64 dims = 1; - - // The data type of the tensor. - // This field MUST have a valid TensorProto.DataType value - optional int32 data_type = 2; - - // For very large tensors, we may want to store them in chunks, in which - // case the following fields will specify the segment that is stored in - // the current TensorProto. - message Segment { - optional int64 begin = 1; - optional int64 end = 2; - } - optional Segment segment = 3; - - // Tensor content must be organized in row-major order. - // - // Depending on the data_type field, exactly one of the fields below with - // name ending in _data is used to store the elements of the tensor. - - // For float and complex64 values - // Complex64 tensors are encoded as a single array of floats, - // with the real components appearing in odd numbered positions, - // and the corresponding imaginary component appearing in the - // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i] - // is encoded as [1.0, 2.0 ,3.0 ,4.0] - // When this field is present, the data_type field MUST be FLOAT or COMPLEX64. - repeated float float_data = 4 [packed = true]; - - // For int32, uint8, int8, uint16, int16, uint4, int4, bool, float8 and float16 values - // float16 and float8 values must be bit-wise converted to an uint16_t prior - // to writing to the buffer. - // uint4 and int4 values must be packed to 4bitx2 prior to writing to the buffer, the first element is stored in - // the 4 LSB and the second element is stored in the 4 MSB. - // When this field is present, the data_type field MUST be - // INT32, INT16, INT8, INT4, UINT16, UINT8, UINT4, BOOL, FLOAT16, BFLOAT16, FLOAT8E4M3FN, FLOAT8E4M3FNUZ, FLOAT8E5M2, FLOAT8E5M2FNUZ - repeated int32 int32_data = 5 [packed = true]; - - // For strings. - // Each element of string_data is a UTF-8 encoded Unicode - // string. No trailing null, no leading BOM. The protobuf "string" - // scalar type is not used to match ML community conventions. - // When this field is present, the data_type field MUST be STRING - repeated bytes string_data = 6; - - // For int64. - // When this field is present, the data_type field MUST be INT64 - repeated int64 int64_data = 7 [packed = true]; - - // Optionally, a name for the tensor. - optional string name = 8; // namespace Value - - // A human-readable documentation for this tensor. Markdown is allowed. - optional string doc_string = 12; - - // Serializations can either use one of the fields above, or use this - // raw bytes field. The only exception is the string case, where one is - // required to store the content in the repeated bytes string_data field. - // - // When this raw_data field is used to store tensor value, elements MUST - // be stored in as fixed-width, little-endian order. - // Floating-point data types MUST be stored in IEEE 754 format. - // Complex64 elements must be written as two consecutive FLOAT values, real component first. - // Complex128 elements must be written as two consecutive DOUBLE values, real component first. - // Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false). - // uint4 and int4 values must be packed to 4bitx2, the first element is stored in the 4 LSB and the second element is stored in the 4 MSB. - // - // Note: the advantage of specific field rather than the raw_data field is - // that in some cases (e.g. int data), protobuf does a better packing via - // variable length storage, and may lead to smaller binary footprint. - // When this field is present, the data_type field MUST NOT be STRING or UNDEFINED - optional bytes raw_data = 9; - - // Data can be stored inside the protobuf file using type-specific fields or raw_data. - // Alternatively, raw bytes data can be stored in an external file, using the external_data field. - // external_data stores key-value pairs describing data location. Recognized keys are: - // - "location" (required) - POSIX filesystem path relative to the directory where the ONNX - // protobuf model was stored - // - "offset" (optional) - position of byte at which stored data begins. Integer stored as string. - // Offset values SHOULD be multiples 4096 (page size) to enable mmap support. - // - "length" (optional) - number of bytes containing data. Integer stored as string. - // - "checksum" (optional) - SHA1 digest of file specified in under 'location' key. - repeated StringStringEntryProto external_data = 13; - - // Location of the data for this tensor. MUST be one of: - // - DEFAULT - data stored inside the protobuf message. Data is stored in raw_data (if set) otherwise in type-specified field. - // - EXTERNAL - data stored in an external location as described by external_data field. - enum DataLocation { - DEFAULT = 0; - EXTERNAL = 1; - } - - // If value not set, data is stored in raw_data (if set) otherwise in type-specified field. - optional DataLocation data_location = 14; - - // For double - // Complex128 tensors are encoded as a single array of doubles, - // with the real components appearing in odd numbered positions, - // and the corresponding imaginary component appearing in the - // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i] - // is encoded as [1.0, 2.0 ,3.0 ,4.0] - // When this field is present, the data_type field MUST be DOUBLE or COMPLEX128 - repeated double double_data = 10 [packed = true]; - - // For uint64 and uint32 values - // When this field is present, the data_type field MUST be - // UINT32 or UINT64 - repeated uint64 uint64_data = 11 [packed = true]; - - // Named metadata values; keys should be distinct. - repeated StringStringEntryProto metadata_props = 16; -} - -// A serialized sparse-tensor value -message SparseTensorProto { - // The sequence of non-default values are encoded as a tensor of shape [NNZ]. - // The default-value is zero for numeric tensors, and empty-string for string tensors. - // values must have a non-empty name present which serves as a name for SparseTensorProto - // when used in sparse_initializer list. - optional TensorProto values = 1; - - // The indices of the non-default values, which may be stored in one of two formats. - // (a) Indices can be a tensor of shape [NNZ, rank] with the [i,j]-th value - // corresponding to the j-th index of the i-th value (in the values tensor). - // (b) Indices can be a tensor of shape [NNZ], in which case the i-th value - // must be the linearized-index of the i-th value (in the values tensor). - // The linearized-index can be converted into an index tuple (k_1,...,k_rank) - // using the shape provided below. - // The indices must appear in ascending order without duplication. - // In the first format, the ordering is lexicographic-ordering: - // e.g., index-value [1,4] must appear before [2,1] - optional TensorProto indices = 2; - - // The shape of the underlying dense-tensor: [dim_1, dim_2, ... dim_rank] - repeated int64 dims = 3; -} - -// Defines a tensor shape. A dimension can be either an integer value -// or a symbolic variable. A symbolic variable represents an unknown -// dimension. -message TensorShapeProto { - message Dimension { - oneof value { - int64 dim_value = 1; - string dim_param = 2; // namespace Shape - }; - // Standard denotation can optionally be used to denote tensor - // dimensions with standard semantic descriptions to ensure - // that operations are applied to the correct axis of a tensor. - // Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition - // for pre-defined dimension denotations. - optional string denotation = 3; - }; - repeated Dimension dim = 1; -} - -// Types -// -// The standard ONNX data types. -message TypeProto { - - message Tensor { - // This field MUST NOT have the value of UNDEFINED - // This field MUST have a valid TensorProto.DataType value - // This field MUST be present for this version of the IR. - optional int32 elem_type = 1; - optional TensorShapeProto shape = 2; - } - - // repeated T - message Sequence { - // The type and optional shape of each element of the sequence. - // This field MUST be present for this version of the IR. - optional TypeProto elem_type = 1; - }; - - // map - message Map { - // This field MUST have a valid TensorProto.DataType value - // This field MUST be present for this version of the IR. - // This field MUST refer to an integral type ([U]INT{8|16|32|64}) or STRING - optional int32 key_type = 1; - // This field MUST be present for this version of the IR. - optional TypeProto value_type = 2; - }; - - // wrapper for Tensor, Sequence, or Map - message Optional { - // The type and optional shape of the element wrapped. - // This field MUST be present for this version of the IR. - // Possible values correspond to OptionalProto.DataType enum - optional TypeProto elem_type = 1; - }; - - - message SparseTensor { - // This field MUST NOT have the value of UNDEFINED - // This field MUST have a valid TensorProto.DataType value - // This field MUST be present for this version of the IR. - optional int32 elem_type = 1; - optional TensorShapeProto shape = 2; - } - - - oneof value { - // The type of a tensor. - Tensor tensor_type = 1; - - // NOTE: DNN-only implementations of ONNX MAY elect to not support non-tensor values - // as input and output to graphs and nodes. These types are needed to naturally - // support classical ML operators. DNN operators SHOULD restrict their input - // and output types to tensors. - - // The type of a sequence. - Sequence sequence_type = 4; - - // The type of a map. - Map map_type = 5; - - // The type of an optional. - Optional optional_type = 9; - - - // Type of the sparse tensor - SparseTensor sparse_tensor_type = 8; - - } - - // An optional denotation can be used to denote the whole - // type with a standard semantic description as to what is - // stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition - // for pre-defined type denotations. - optional string denotation = 6; -} - -// Operator Sets -// -// OperatorSets are uniquely identified by a (domain, opset_version) pair. -message OperatorSetIdProto { - // The domain of the operator set being identified. - // The empty string ("") or absence of this field implies the operator - // set that is defined as part of the ONNX specification. - // This field MUST be present in this version of the IR when referring to any other operator set. - optional string domain = 1; - - // The version of the operator set being identified. - // This field MUST be present in this version of the IR. - optional int64 version = 2; -} - -// Operator/function status. -enum OperatorStatus { - EXPERIMENTAL = 0; - STABLE = 1; -} - -message FunctionProto { - // The name of the function, similar to op_type in NodeProto. - // This is part of the unique-id (domain, name, overload) of FunctionProtos in a model. - optional string name = 1; - - // Deprecated since IR Version 8 - // optional int64 since_version = 2; - reserved 2; - reserved "since_version"; - - // Deprecated since IR Version 8 - // optional OperatorStatus status = 3; - reserved 3; - reserved "status"; - - // The inputs and outputs of the function. - repeated string input = 4; - repeated string output = 5; - - // The attribute parameters of the function. - // It is for function parameters without default values. - repeated string attribute = 6; - - // The attribute protos of the function. - // It is for function attributes with default values. - // A function attribute shall be represented either as - // a string attribute or an AttributeProto, not both. - repeated AttributeProto attribute_proto = 11; - - // The nodes in the function. - repeated NodeProto node = 7; - // A human-readable documentation for this function. Markdown is allowed. - optional string doc_string = 8; - - // The OperatorSets this function body (graph) relies on. - // - // All nodes in the function body (graph) will bind against the operator - // with the same-domain/same-op_type operator with the HIGHEST version - // in the referenced operator sets. This means at most one version can be relied - // for one domain. - // - // The operator sets imported by FunctionProto should be compatible with the ones - // imported by ModelProto. Example, if same operator set say 'A' is imported by FunctionProto - // and ModelProto then versions for the operator set may be different but, - // the operator schema returned for op_type, domain, version combination - // for both the versions should be same. - - repeated OperatorSetIdProto opset_import = 9; - - // The domain which this function belongs to. - // This is part of the unique-id (domain, name, overload) of FunctionProtos in a model. - optional string domain = 10; - - // The overload identifier of the function. - // This is part of the unique-id (domain, name, overload) of FunctionProtos in a model. - optional string overload = 13; - - // Information for the values in the function. The ValueInfoProto.name's - // must be distinct and refer to names in the function (including inputs, - // outputs, and intermediate values). It is optional for a value to appear - // in value_info list. - repeated ValueInfoProto value_info = 12; - - // Named metadata values; keys should be distinct. - repeated StringStringEntryProto metadata_props = 14; -} - -// For using protobuf-lite -option optimize_for = LITE_RUNTIME; - diff --git a/docker/rocm/migraphx/onnx/onnx_parser.cpp b/docker/rocm/migraphx/onnx/onnx_parser.cpp deleted file mode 100644 index cf81e9217..000000000 --- a/docker/rocm/migraphx/onnx/onnx_parser.cpp +++ /dev/null @@ -1,713 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_ONNX_PARSER) - -static shape shape_from_dyn_dims(shape::type_t shape_type, - const std::vector& dyn_dims) -{ - if(std::all_of(dyn_dims.begin(), dyn_dims.end(), [](auto dd) { return dd.is_fixed(); })) - { - std::vector dims; - std::transform(dyn_dims.cbegin(), dyn_dims.cend(), std::back_inserter(dims), [](auto d) { - return d.max; - }); - return {shape_type, dims}; - } - return {shape_type, dyn_dims}; -} - -static onnx_parser::attribute_map get_attributes(const onnx::NodeProto& node) -{ - std::unordered_map result; - for(auto&& attr : node.attribute()) - { - result[attr.name()] = attr; - } - return result; -} - -static literal -create_literal(shape::type_t shape_type, const std::vector& dims, const char* data) -{ - // empty input - auto elem_num = - std::accumulate(dims.begin(), dims.end(), std::size_t(1), std::multiplies()); - if(elem_num == 0) - { - return literal{shape_type}; - } - - // in case of scalar constants in onnx file, use dims=1 to fill initializer data - if(dims.empty()) - return literal{{shape_type}, data}; - return literal{{shape_type, dims}, data}; -} - -template {})> -static literal create_literal(shape::type_t shape_type, const std::vector& dims, T data) -{ - // empty input - auto elem_num = - std::accumulate(dims.begin(), dims.end(), std::size_t(1), std::multiplies()); - if(elem_num == 0) - { - return literal{shape_type}; - } - - // scalar input - if(dims.empty()) - return literal{{shape_type}, data.begin(), data.end()}; - return literal{{shape_type, dims}, data.begin(), data.end()}; -} - -template -static literal from_repeated(shape::type_t t, const T& r) -{ - std::size_t size = r.size(); - return literal{{t, {size}}, r.begin(), r.end()}; -} - -instruction_ref onnx_parser::node_info::make_contiguous(instruction_ref ins) const -{ - auto attr = ins->get_operator().to_value(); - std::string key = "require_std_shape"; - if((attr.get(key, false)) or (not ins->get_shape().standard())) - { - return add_instruction(make_op("contiguous"), ins); - } - - return ins; -} - -instruction_ref onnx_parser::node_info::add_bias(const std::vector& args, - instruction_ref curr_ins, - uint64_t axis) const -{ - if(args.size() == 3) - { - instruction_ref bias_bcast; - // if curr_ins has a dynamic output shape use 2 input broadcast - if(curr_ins->get_shape().dynamic()) - { - bias_bcast = - mod->add_instruction(make_op("broadcast", {{"axis", axis}}), args[2], curr_ins); - } - else - { - bias_bcast = mod->add_instruction( - make_op("broadcast", {{"axis", axis}, {"out_lens", curr_ins->get_shape().lens()}}), - args[2]); - } - return mod->add_instruction(make_op("add"), curr_ins, bias_bcast); - } - return curr_ins; -} - -instruction_ref onnx_parser::node_info::add_broadcastable_binary_op(const std::string& op_name, - instruction_ref arg0, - instruction_ref arg1) const -{ - return this->add_common_op(op_name, arg0, arg1); -} - -/** - * @brief A wrapper for insert_common_args(), which constructs an argument list - * and inserts multibroadcast and convert ops to match inputs to a common shape and type - * as required. The requested operation is placed after the added multibroadcast and convert ops, - * if any, so that their results are transparent to the programmer. - * - * Use add_common_op() to match input sizes when inputs may be - * either static or dynamic. - * - * @param op_name string; Name of operation (op) to add; valid names are the same as - * for make_op() - * - * @param inputs vector of instruction_ref. List of instructions for the new - * operator. Multibroadcast and convert operations, if needed, are deduced from these too. - * - * @return instruction_ref Returns an instruction_ref which is the result of the requested - * operation. - * - */ -instruction_ref onnx_parser::node_info::add_common_op(const std::string& op_name, - std::vector inputs) const -{ - return migraphx::add_common_op(*mod, make_op(op_name), std::move(inputs)); -} - -instruction_ref -onnx_parser::node_info::add_instruction(const operation& op, - const std::vector& args) const -{ - return mod->add_instruction(op, args); -} - -instruction_ref onnx_parser::node_info::add_instruction(const operation& op, - const std::vector& args, - const std::vector& mods) const -{ - return mod->add_instruction(op, args, mods); -} - -instruction_ref onnx_parser::node_info::add_literal(literal l) const -{ - return mod->add_literal(std::move(l)); -} - -onnx_parser::onnx_parser() -{ - // Add all registered op parsers - for(auto&& name : get_op_parsers()) - ops.emplace(name, get_op_parser(name)); -} - -operation onnx_parser::load(const std::string& name, const node_info& info) const -{ - auto op = make_op(name); - auto v = op.to_value(); - for(auto&& x : v) - { - if(info.attributes.count(x.get_key()) == 0) - continue; - literal s = parse_value(info.attributes.at(x.get_key())); - if(x.is_array()) - { - std::vector values; - s.visit([&](auto y) { - std::transform(y.begin(), y.end(), std::back_inserter(values), [](auto z) { - return value(z); - }); - }); - x = values; - } - else - { - s.visit([&](auto y) { x = y.front(); }); - } - } - op.from_value(v); - return op; -} - -void onnx_parser::parse_undefined(module* mod, const std::string& name) -{ - if(not contains(instructions, name)) - { - auto ins = mod->add_instruction(make_op("undefined")); - instructions[name] = ins; - } -} - -void onnx_parser::parse_from(std::istream& is, std::string name) -{ - auto* mm = prog.get_main_module(); - this->filename = std::move(name); - auto parent_path = fs::path(this->filename).parent_path(); - if(not parent_path.empty()) - this->path = parent_path.string(); - - onnx::ModelProto model; - if(model.ParseFromIstream(&is)) - { - auto version = get_opset_version(model); - opset_version = (version == -1) ? opset_version : version; - - if(model.has_graph()) - { - (void)this->parse_graph(mm, model.graph()); - } - } - else - { - MIGRAPHX_THROW("PARSE_FROM: Failed reading onnx file: " + this->filename); - } -} - -void onnx_parser::parse_from(const void* data, std::size_t size) -{ - auto* mm = prog.get_main_module(); - onnx::ModelProto model; - if(model.ParseFromArray(data, size)) - { - auto version = get_opset_version(model); - opset_version = (version == -1) ? opset_version : version; - - if(model.has_graph()) - { - (void)this->parse_graph(mm, model.graph()); - } - } - else - { - MIGRAPHX_THROW("Failed reading onnx file."); - } -} - -int64_t onnx_parser::get_opset_version(const onnx::ModelProto& model) -{ - const auto& opset_import = model.opset_import(); - int64_t version = -1; - for(const auto& opset : opset_import) - { - if(opset.has_version()) - { - version = std::max(version, opset.version()); - } - } - - return version; -} - -void print_added_instructions(module* mod, - const std::vector& args, - const std::vector& result) -{ - // Print instructions added by the parser not in args - std::vector added_instructions; - fix([&](auto self, auto r) { - for(auto ins : r) - { - if(contains(args, ins)) - continue; - if(contains(added_instructions, ins)) - continue; - self(ins->inputs()); - added_instructions.push_back(ins); - } - })(result); - mod->debug_print(added_instructions); -} - -static bool is_type_packed_int4(const onnx::TensorProto& t) -{ - return t.data_type() == onnx::TensorProto::INT4 or t.data_type() == onnx::TensorProto::UINT4; -} - -std::unordered_map -parse_intializer(const onnx_parser& parser, module* mod, const onnx::GraphProto& graph) -{ - std::unordered_map mod_insts; - for(auto&& f : graph.initializer()) - { - if(enabled(MIGRAPHX_TRACE_ONNX_PARSER{})) - std::cout << "initializer: " << f.name() << std::endl; - // backup instructions in parent mod - auto pt = parser.parse_tensor(f); - auto lit = mod->add_literal(pt); - - if(is_type_packed_int4(f)) - lit = mod->add_instruction(migraphx::make_op("unpack_int4"), lit); - - mod_insts[f.name()] = lit; - if(enabled(MIGRAPHX_TRACE_ONNX_PARSER{})) - mod->debug_print(mod_insts[f.name()]); - } - return mod_insts; -} - -std::unordered_map -parse_inputs(const onnx_parser& parser, - module* mod, - const onnx::GraphProto& graph, - std::unordered_map mod_insts) -{ - for(auto&& input : graph.input()) - { - const std::string& name = input.name(); - // input not in initializer_data, so it is a real input - if(not contains(mod_insts, name)) - { - // ONNX specification does not specify how to deal with the - // scenario that a nested subgraph contains a parameter with the - // name existed in its parent graph. - // In the current implementation, MIGraphX throws an exception for that. - if(contains(parser.instructions, name)) - { - MIGRAPHX_THROW("module \"" + mod->name() + "\" has parameter name \"" + name + - "\" existing in parent graph!"); - } - - shape s; - if(parser.map_input_dims.count(name) > 0) - { - std::vector dims = parser.map_input_dims.at(name); - s = parser.parse_type(input.type(), dims); - } - else if(parser.map_dyn_input_dims.count(name) > 0) - { - shape::type_t shape_type = get_type(input.type().tensor_type().elem_type()); - s = shape_from_dyn_dims(shape_type, parser.map_dyn_input_dims.at(name)); - } - else - { - s = parser.parse_type(input.type()); - } - mod_insts[name] = mod->add_parameter(name, s); - } - } - return mod_insts; -} - -std::vector -onnx_parser::parse_graph(module* mod, const onnx::GraphProto& graph, bool inlining) -{ - std::unordered_map mod_insts = - parse_intializer(*this, mod, graph); - - mod_insts = parse_inputs(*this, mod, graph, mod_insts); - - std::copy(mod_insts.begin(), mod_insts.end(), std::inserter(instructions, instructions.end())); - - for(auto&& node : graph.node()) - { - if(enabled(MIGRAPHX_TRACE_ONNX_PARSER{})) - std::cout << "operator: " << node.op_type() << std::endl; - - std::vector args; - for(auto&& input : node.input()) - { - if(input.empty()) - { - this->parse_undefined(mod, input); - } - if(instructions.count(input) == 0) - { - MIGRAPHX_THROW("PARSE_GRAPH: invalid onnx file. Input \"" + input + - "\" is unavailable due to unordered nodes!"); - } - args.push_back(instructions.at(input)); - } - - std::vector result; - std::size_t output_num = node.output().size(); - if(ops.count(node.op_type()) == 0) - { - if(skip_unknown_operators) - result.push_back(mod->add_instruction(op::unknown{node.op_type()}, args)); - else - MIGRAPHX_THROW("Unknown operator: " + node.op_type()); - } - else - { - std::string node_name = node.op_type() + "_" + std::to_string(mod->size()); - result = ops[node.op_type()]( - *this, {get_attributes(node), output_num, node_name, mod}, args); - } - - output_num = std::min(output_num, result.size()); - std::transform(node.output().begin(), - node.output().begin() + output_num, - result.begin(), - std::inserter(instructions, instructions.end()), - [](auto&& x, auto&& y) { return std::make_pair(x, y); }); - - if(enabled(MIGRAPHX_TRACE_ONNX_PARSER{})) - { - print_added_instructions(mod, args, result); - } - } - - // Find instructions corresponding to the output - auto prog_output = graph.output(); - std::vector all_output_names; - std::vector prog_output_names; - std::transform(prog_output.begin(), - prog_output.end(), - std::back_inserter(all_output_names), - [](auto& node) { return node.name(); }); - std::copy_if( - all_output_names.begin(), - all_output_names.end(), - std::back_inserter(prog_output_names), - [&](const auto& name) { return not(name.empty() or instructions.count(name) == 0); }); - - std::vector output_ins; - std::transform(prog_output_names.begin(), - prog_output_names.end(), - std::back_inserter(output_ins), - [&](const auto& name) { return instructions[name]; }); - - if(not inlining) - { - // add the return instuction - mod->add_return(output_ins); - - // Remove instructions added in module (this is turned off for subgraph inlining) - erase_if(instructions, [&](auto&& p) { return mod->has_instruction(p.second); }); - } - - return output_ins; -} - -literal onnx_parser::parse_value(const onnx::AttributeProto& attr) const -{ - switch(attr.type()) - { - case onnx::AttributeProto::FLOAT: return literal{attr.f()}; - case onnx::AttributeProto::INT: return literal{attr.i()}; - case onnx::AttributeProto::TENSOR: return parse_tensor(attr.t()); - case onnx::AttributeProto::FLOATS: return from_repeated(shape::float_type, attr.floats()); - case onnx::AttributeProto::INTS: return from_repeated(shape::int64_type, attr.ints()); - case onnx::AttributeProto::UNDEFINED: - case onnx::AttributeProto::GRAPH: - case onnx::AttributeProto::STRING: - case onnx::AttributeProto::STRINGS: - case onnx::AttributeProto::TENSORS: - case onnx::AttributeProto::SPARSE_TENSOR: - case onnx::AttributeProto::SPARSE_TENSORS: - case onnx::AttributeProto::TYPE_PROTOS: - case onnx::AttributeProto::TYPE_PROTO: - case onnx::AttributeProto::GRAPHS: return {}; - } - MIGRAPHX_THROW("PARSE_VALUE: Invalid attribute type " + std::to_string(attr.type())); -} - -static shape parse_tensor_shape(const onnx::TensorProto& t) -{ - std::vector dims(t.dims().begin(), t.dims().end()); - if(is_type_packed_int4(t)) - { - auto dim_n = dims.back(); - if(dim_n > 0 and (dim_n % 2 == 0)) - dims.back() = dim_n / 2; // int4-packed dimension converted to int8-sized units - else - MIGRAPHX_THROW("Int4: currently supports only even-sized packed tensors"); - } - return shape{get_type(t.data_type()), dims}; -} - -literal onnx_parser::parse_tensor(const onnx::TensorProto& t) const -{ - auto tensor_shape = parse_tensor_shape(t); - const auto& dims = tensor_shape.lens(); - auto type = tensor_shape.type(); - auto external_data = t.external_data(); - - if(not external_data.empty()) - { - const std::string& data_file = external_data.at(0).value(); - size_t num_data_fields = external_data.size(); - size_t offset = 0; - size_t nbytes = tensor_shape.bytes(); - - if(num_data_fields > 1) // if offset field is present - { - offset = std::stoull(t.external_data().at(1).value()); - } - if(num_data_fields > 2) // if nbytes field is present - { - nbytes = std::stoull(t.external_data().at(2).value()); - } - std::vector raw_buffer; - if(not external_data_path.empty()) - { - raw_buffer = read_buffer(fs::path{external_data_path} / data_file, offset, nbytes); - } - else - { - raw_buffer = read_buffer(path / data_file, offset, nbytes); - } - std::string s(raw_buffer.begin(), raw_buffer.end()); - return create_literal(type, dims, s.data()); - } - - if(t.has_raw_data()) - { - const std::string& s = t.raw_data(); - return create_literal(type, dims, s.data()); - } - - switch(t.data_type()) - { - case onnx::TensorProto::BOOL: return create_literal(shape::bool_type, dims, t.int32_data()); - - // INT4 or UINT4 operate as 8-bit buffers: - case onnx::TensorProto::INT4: return create_literal(shape::int8_type, dims, t.int32_data()); - case onnx::TensorProto::UINT4: return create_literal(shape::uint8_type, dims, t.int32_data()); - - case onnx::TensorProto::INT8: return create_literal(shape::int8_type, dims, t.int32_data()); - case onnx::TensorProto::UINT8: return create_literal(shape::uint8_type, dims, t.int32_data()); - - case onnx::TensorProto::INT16: return create_literal(shape::int16_type, dims, t.int32_data()); - case onnx::TensorProto::UINT16: return create_literal(shape::uint16_type, dims, t.int32_data()); - - case onnx::TensorProto::INT32: return create_literal(shape::int32_type, dims, t.int32_data()); - case onnx::TensorProto::UINT32: - return create_literal(shape::uint32_type, dims, t.uint64_data()); - - case onnx::TensorProto::INT64: return create_literal(shape::int64_type, dims, t.int64_data()); - case onnx::TensorProto::UINT64: - return create_literal(shape::uint64_type, dims, t.uint64_data()); - - case onnx::TensorProto::FLOAT16: { - std::vector data_uint16(t.int32_data().begin(), t.int32_data().end()); - std::vector data_half; - std::transform(data_uint16.begin(), - data_uint16.end(), - std::back_inserter(data_half), - [](uint16_t raw_val) { return *reinterpret_cast(&raw_val); }); - return create_literal(shape::half_type, dims, data_half); - } - - case onnx::TensorProto::DOUBLE: - return create_literal(shape::double_type, dims, t.double_data()); - - case onnx::TensorProto::FLOAT: return create_literal(shape::float_type, dims, t.float_data()); - - case onnx::TensorProto::FLOAT8E4M3FNUZ: { - std::vector data_int32(t.int32_data().begin(), t.int32_data().end()); - std::vector data_fp8; - std::transform(data_int32.begin(), - data_int32.end(), - std::back_inserter(data_fp8), - [](float raw_val) { return migraphx::fp8::fp8e4m3fnuz{raw_val}; }); - return create_literal(shape::fp8e4m3fnuz_type, dims, data_fp8); - } - - case onnx::TensorProto::FLOAT8E5M2FNUZ: - case onnx::TensorProto::FLOAT8E5M2: - case onnx::TensorProto::FLOAT8E4M3FN: - case onnx::TensorProto::UNDEFINED: - case onnx::TensorProto::STRING: - case onnx::TensorProto::COMPLEX64: - case onnx::TensorProto::COMPLEX128: throw std::runtime_error(""); - } - MIGRAPHX_THROW("PARSE_TENSOR: Invalid tensor type"); -} - -shape onnx_parser::parse_type(const onnx::TypeProto& t) const -{ - shape::type_t shape_type = get_type(t.tensor_type().elem_type()); - - std::vector dynamic_dims; - auto&& tensor_dims = t.tensor_type().shape().dim(); - std::transform(tensor_dims.begin(), - tensor_dims.end(), - std::back_inserter(dynamic_dims), - [&](auto&& d) -> shape::dynamic_dimension { - if(d.has_dim_param()) - { - const auto& dim_param = d.dim_param(); - if(contains(dim_params, dim_param)) - { - return dim_params.at(dim_param); - } - } - if(d.has_dim_value()) - { - if(static_cast(d.dim_value()) <= 0) - { - return default_dyn_dim_value; - } - std::size_t tmp = d.dim_value(); - return {tmp, tmp}; - } - else - { - return default_dyn_dim_value; - } - }); - - if(dynamic_dims.empty()) - { - return {shape_type}; - } - return shape_from_dyn_dims(shape_type, dynamic_dims); -} - -shape onnx_parser::parse_type(const onnx::TypeProto& t, - const std::vector& input_dims) const -{ - shape::type_t shape_type = get_type(t.tensor_type().elem_type()); - if(input_dims.empty()) - return {shape_type}; - return {shape_type, input_dims}; -} - -shape::type_t get_type(int dtype) -{ - switch(dtype) - { - case 1: return shape::float_type; - case 2: return shape::uint8_type; - case 3: return shape::int8_type; - case 4: return shape::uint16_type; - case 5: return shape::int16_type; - case 6: return shape::int32_type; - case 7: return shape::int64_type; - case 9: return shape::bool_type; - case 10: return shape::half_type; - case 11: return shape::double_type; - case 12: return shape::uint32_type; - case 13: return shape::uint64_type; - case 18: return shape::fp8e4m3fnuz_type; - case 21: return shape::uint8_type; - case 22: return shape::int8_type; - case 14: - case 15: - case 16: return shape::bf16_type; - case 17: - case 19: - case 20: - default: { - MIGRAPHX_THROW("Prototensor data type " + std::to_string(dtype) + " not supported"); - } - } -} - -bool is_type_float(shape::type_t dtype) -{ - bool r = false; - if(dtype == shape::float_type or dtype == shape::double_type or dtype == shape::half_type or - dtype == shape::bf16_type) - { - r = true; - } - return r; -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/op_parser.cpp b/docker/rocm/migraphx/onnx/op_parser.cpp deleted file mode 100644 index 24ab27c45..000000000 --- a/docker/rocm/migraphx/onnx/op_parser.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -std::unordered_map& op_parser_map() -{ - static std::unordered_map m; // NOLINT - return m; -} - -void register_op_parser(const std::string& name, onnx_parser::op_func f) -{ - op_parser_map()[name] = std::move(f); -} -onnx_parser::op_func get_op_parser(const std::string& name) { return op_parser_map().at(name); } -std::vector get_op_parsers() -{ - std::vector result; - std::transform(op_parser_map().begin(), - op_parser_map().end(), - std::back_inserter(result), - [&](auto&& p) { return p.first; }); - std::sort(result.begin(), result.end()); - return result; -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/padding.cpp b/docker/rocm/migraphx/onnx/padding.cpp deleted file mode 100644 index 409d81e76..000000000 --- a/docker/rocm/migraphx/onnx/padding.cpp +++ /dev/null @@ -1,179 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void cal_auto_padding_size(onnx_parser::node_info info, - value& v, - const std::vector& k_lens, - const std::vector& dilation, - const std::vector& in_lens, - std::vector& paddings) -{ - size_t kdims = in_lens.size() - 2; - assert(k_lens.size() == kdims and dilation.size() == kdims); - - if(not contains(info.attributes, "auto_pad")) - { - return; - } - - auto auto_pad = to_upper(info.attributes["auto_pad"].s()); - if(auto_pad.find("SAME") != std::string::npos) - { - bool is_same_upper = (auto_pad.find("SAME_UPPER") != std::string::npos); - paddings.resize(2 * kdims); - - for(size_t i = 0; i < paddings.size() / 2; i++) - { - calculate_padding(i, - paddings, - in_lens[i + 2], - v["stride"][i].to(), - dilation[i], - k_lens[i], - is_same_upper); - } - } -} - -bool is_asym_padding(const std::vector& padding) -{ - assert(padding.size() % 2 == 0); - size_t pad_ndims = padding.size() / 2; - - for(size_t i = 0; i < pad_ndims; i++) - { - if(padding[i] != padding[i + pad_ndims]) - { - return true; - } - } - return false; -} - -void check_padding_mode(const onnx_parser::node_info& info, const std::string& onnx_name) -{ - // ensure pads availabe only when auto_pad is "NOT_SET" - if(contains(info.attributes, "pads") and contains(info.attributes, "auto_pad")) - { - auto s = info.attributes.at("auto_pad").s(); - if(to_upper(s) != "NOTSET") - { - MIGRAPHX_THROW("PARSE_" + to_upper(onnx_name) + - ": auto_pad and padding cannot be specified simultaneously"); - } - } -} - -static void -tune_padding_to_symmetric(int64_t& left, int64_t& right, const int stride, int64_t& s_start) -{ - s_start = 0; - if(left > right) - { - right = left; - } - else if(left < right) - { - auto diff = right - left; - s_start = (diff + stride - 1) / stride; - left = left + s_start * stride; - right = left; - } -} - -void tune_padding_size(const value& v, - std::vector& padding, - int count_include_pad, - std::vector& s_start) -{ - // maxpooling or count_include_pad is 1, no change is required. - if(v.at("mode").to() == op::pooling_mode::max or count_include_pad == 1) - { - return; - } - - // if padding is symmetric, return directly - if(not is_asym_padding(padding)) - { - return; - } - - // asymmetric padding, make it symmetric - std::size_t n_dims = padding.size() / 2; - s_start.resize(n_dims); - for(std::size_t i = 0; i < n_dims; ++i) - { - tune_padding_to_symmetric( - padding[i], padding[i + n_dims], v.at("stride")[i].to(), s_start[i]); - } -} - -void check_asym_padding(const onnx_parser::node_info& info, - instruction_ref& ins, - const std::vector& padding, - value& v, - int count_include_pad, - float pad_val) -{ - size_t pad_ndims = padding.size() / 2; - auto left_pad_it = padding.begin(); - auto right_pad_it = left_pad_it + pad_ndims; - - if(count_include_pad == 1) - { - std::vector asym_pads{0, 0, 0, 0}; // don't pad N and C - // add left pads - asym_pads.insert(asym_pads.begin() + 2, left_pad_it, right_pad_it); - // add right pads - asym_pads.insert(asym_pads.begin() + pad_ndims + 4, right_pad_it, padding.end()); - ins = info.add_instruction(make_op("pad", {{"pads", asym_pads}, {"value", pad_val}}), ins); - std::vector new_padding(padding.size()); - // subtract asym padding originally found from parsing the operator - std::transform(padding.begin(), - left_pad_it, - asym_pads.begin() + 2, - new_padding.begin(), - std::minus()); - std::transform(right_pad_it, - padding.end(), - asym_pads.begin() + pad_ndims + 4, - new_padding.begin() + pad_ndims, - std::minus()); - v["padding"] = new_padding; - } -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_arg_op.cpp b/docker/rocm/migraphx/onnx/parse_arg_op.cpp deleted file mode 100644 index 905496461..000000000 --- a/docker/rocm/migraphx/onnx/parse_arg_op.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_arg_op : op_parser -{ - std::vector operators() const { return {{"ArgMax", "argmax"}, {"ArgMin", "argmin"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - const std::vector& args) const - { - int64_t axis = 0; - if(contains(info.attributes, "axis")) - { - axis = static_cast(parser.parse_value(info.attributes.at("axis")).at()); - } - - int keep_dims = 1; - if(contains(info.attributes, "keepdims")) - { - keep_dims = parser.parse_value(info.attributes.at("keepdims")).at(); - } - - bool select_last_index = false; - if(contains(info.attributes, "select_last_index")) - { - select_last_index = static_cast( - parser.parse_value(info.attributes.at("select_last_index")).at()); - } - - if(keep_dims == 0) - { - auto ins = info.add_instruction( - make_op(opd.op_name, {{"axis", axis}, {"select_last_index", select_last_index}}), - args); - return info.add_instruction(make_op("squeeze", {{"axes", {axis}}}), ins); - } - else - { - return info.add_instruction( - make_op(opd.op_name, {{"axis", axis}, {"select_last_index", select_last_index}}), - args); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_aten.cpp b/docker/rocm/migraphx/onnx/parse_aten.cpp deleted file mode 100644 index 79126d20a..000000000 --- a/docker/rocm/migraphx/onnx/parse_aten.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -enum class reduce_mode_t -{ - sum = 0, - mean = 1, - max = 2 -}; - -struct parse_aten : op_parser -{ - std::vector operators() const { return {{"ATen"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - if(contains(info.attributes, "operator")) - { - auto op_name = info.attributes.at("operator").s(); - if(op_name.find("embedding_bag") != std::string::npos) - { - return parse_embedding_bag(info, std::move(args)); - } - } - MIGRAPHX_THROW("PARSE_ATEN: unsupported custom operator"); - } - - instruction_ref parse_embedding_bag(onnx_parser::node_info info, - std::vector args) const - { - if(args[2]->get_shape().elements() != 1) - MIGRAPHX_THROW("PARSE_EMBEDDING_BAG: MIGraphX only supports offsets of size 1"); - reduce_mode_t reduce_mode = reduce_mode_t::sum; - if(contains(info.attributes, "mode")) - { - reduce_mode = static_cast(info.attributes.at("mode").i()); - } - - auto l0 = info.add_instruction(make_op("gather"), args[0], args[1]); - switch(reduce_mode) - { - case reduce_mode_t::sum: - l0 = info.add_instruction(make_op("reduce_sum", {{"axes", {0}}}), l0); - break; - case reduce_mode_t::mean: - l0 = info.add_instruction(make_op("reduce_mean", {{"axes", {0}}}), l0); - break; - case reduce_mode_t::max: - l0 = info.add_instruction(make_op("reduce_max", {{"axes", {0}}}), l0); - break; - } - return l0; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_batchnorm.cpp b/docker/rocm/migraphx/onnx/parse_batchnorm.cpp deleted file mode 100644 index 0f07df678..000000000 --- a/docker/rocm/migraphx/onnx/parse_batchnorm.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_batchnorm : op_parser -{ - std::vector operators() const { return {{"BatchNormalization"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - float epsilon = 1e-5f; - if(contains(info.attributes, "epsilon")) - { - epsilon = parser.parse_value(info.attributes.at("epsilon")).at(); - } - auto x_lens = args[0]->get_shape().max_lens(); - auto x_type = args[0]->get_shape().type(); - - if(std::any_of(args.cbegin() + 1, args.cend(), [](auto a) { - return a->get_shape().lens().size() != 1; - })) - { - MIGRAPHX_THROW("PARSE_BATCHNORM: argument scale, bias, mean, or var rank != 1"); - } - - auto x_rank = x_lens.size(); - if(x_rank == 1 or x_rank == 2) - { - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {epsilon}}); - auto x_sub_mean = info.add_broadcastable_binary_op("sub", args[0], args[3]); - auto var_eps = info.add_broadcastable_binary_op("add", args[4], eps); - auto rsqrt = info.add_instruction(make_op("rsqrt"), var_eps); - auto mul0 = info.add_broadcastable_binary_op("mul", args[1], rsqrt); - auto r0 = info.add_broadcastable_binary_op("mul", x_sub_mean, mul0); - return info.add_broadcastable_binary_op("add", r0, args[2]); - } - else if(x_rank > 2) - { - // unsqueeze tensors of shape (C) to broadcast correctly - std::vector unsqueeze_axes(x_lens.size() - 2); - std::iota(unsqueeze_axes.begin(), unsqueeze_axes.end(), 1); - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {epsilon}}); - auto scale_unsqueeze = info.add_instruction( - migraphx::make_op("unsqueeze", {{"axes", unsqueeze_axes}}), args[1]); - auto bias_unsqueeze = info.add_instruction( - migraphx::make_op("unsqueeze", {{"axes", unsqueeze_axes}}), args[2]); - auto mean_unsqueeze = info.add_instruction( - migraphx::make_op("unsqueeze", {{"axes", unsqueeze_axes}}), args[3]); - auto var_unsqueeze = info.add_instruction( - migraphx::make_op("unsqueeze", {{"axes", unsqueeze_axes}}), args[4]); - auto x_sub_mean = info.add_broadcastable_binary_op("sub", args[0], mean_unsqueeze); - auto var_eps = info.add_broadcastable_binary_op("add", var_unsqueeze, eps); - auto rsqrt = info.add_instruction(make_op("rsqrt"), var_eps); - auto mul0 = info.add_broadcastable_binary_op("mul", scale_unsqueeze, rsqrt); - auto r0 = info.add_broadcastable_binary_op("mul", x_sub_mean, mul0); - return info.add_broadcastable_binary_op("add", r0, bias_unsqueeze); - } - else - { - // rank == 0 - MIGRAPHX_THROW("PARSE_BATCHNORM: rank " + std::to_string(x_lens.size()) + - " input tensor, unhandled data format"); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_binary_op.cpp b/docker/rocm/migraphx/onnx/parse_binary_op.cpp deleted file mode 100644 index 58d01a566..000000000 --- a/docker/rocm/migraphx/onnx/parse_binary_op.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_binary_op : op_parser -{ - std::vector operators() const - { - return {{"Add", "add"}, - {"Div", "div"}, - {"And", "logical_and"}, - {"Or", "logical_or"}, - {"Xor", "logical_xor"}, - {"BitwiseAnd", "bitwise_and"}, - {"Mul", "mul"}, - {"PRelu", "prelu"}, - {"Sub", "sub"}}; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - if(args.size() != 2) - MIGRAPHX_THROW("binary operators should have 2 operands"); - if(contains(info.attributes, "broadcast") and contains(info.attributes, "axis")) - { - uint64_t broadcasted = - parser.parse_value(info.attributes.at("broadcast")).at(); - if(broadcasted != 0) - { - if(std::any_of( - args.cbegin(), args.cend(), [](auto a) { return a->get_shape().dynamic(); })) - { - MIGRAPHX_THROW( - "Binary op broadcast attribute not supported for dynamic input shapes"); - } - uint64_t axis = parser.parse_value(info.attributes.at("axis")).at(); - auto l = info.add_instruction( - make_op("broadcast", - {{"axis", axis}, {"out_lens", args[0]->get_shape().lens()}}), - args[1]); - return info.add_instruction(make_op(opd.op_name), args[0], l); - } - return info.add_instruction(make_op(opd.op_name), args); - } - else - { - return info.add_broadcastable_binary_op(opd.op_name, args[0], args[1]); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_cast.cpp b/docker/rocm/migraphx/onnx/parse_cast.cpp deleted file mode 100644 index 6c2de3fac..000000000 --- a/docker/rocm/migraphx/onnx/parse_cast.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_cast : op_parser -{ - std::vector operators() const { return {{"Cast"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - const std::vector& args) const - { - if(not contains(info.attributes, "to")) - { - MIGRAPHX_THROW("PARSE_CAST: missing to type attribute!"); - } - - int to_type = parser.parse_value(info.attributes.at("to")).at(); - shape::type_t type = get_type(to_type); - return info.add_instruction(make_op("convert", {{"target_type", type}}), args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_castlike.cpp b/docker/rocm/migraphx/onnx/parse_castlike.cpp deleted file mode 100644 index 61d1fb27b..000000000 --- a/docker/rocm/migraphx/onnx/parse_castlike.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_castlike : op_parser -{ - std::vector operators() const { return {{"CastLike"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - const std::vector& args) const - { - if(not(args.size() == 2)) - { - MIGRAPHX_THROW("PARSE_CASTLIKE: CastLike must have exactly 2 inputs!"); - } - shape::type_t target_type = args[1]->get_shape().type(); - return info.add_instruction(make_op("convert", {{"target_type", target_type}}), args[0]); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_celu.cpp b/docker/rocm/migraphx/onnx/parse_celu.cpp deleted file mode 100644 index 3bd8fd62e..000000000 --- a/docker/rocm/migraphx/onnx/parse_celu.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_celu : op_parser -{ - std::vector operators() const { return {{"Celu"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser&, - const onnx_parser::node_info& info, - std::vector args) const - { - float alpha = 1.0; - if(contains(info.attributes, "alpha")) - { - alpha = info.attributes.at("alpha").f(); - } - if(float_equal(alpha, 0.0f)) - { - MIGRAPHX_THROW("CELU: alpha is zero (division by zero)"); - } - - auto input_lens = args[0]->get_shape().lens(); - auto input_type = args[0]->get_shape().type(); - if(input_type != migraphx::shape::float_type) - { - MIGRAPHX_THROW("CELU: input tensor not float type"); - } - auto zero_lit = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {0.}})); - auto one_lit = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {1.}})); - auto alpha_lit = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {alpha}})); - auto linear_part = info.add_instruction(migraphx::make_op("max"), zero_lit, args[0]); - auto divi = info.add_instruction(migraphx::make_op("div"), args[0], alpha_lit); - auto expo = info.add_instruction(migraphx::make_op("exp"), divi); - auto sub = info.add_instruction(migraphx::make_op("sub"), expo, one_lit); - auto mul = info.add_instruction(migraphx::make_op("mul"), alpha_lit, sub); - auto exp_part = info.add_instruction(migraphx::make_op("min"), zero_lit, mul); - return info.add_instruction(migraphx::make_op("add"), linear_part, exp_part); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_clip.cpp b/docker/rocm/migraphx/onnx/parse_clip.cpp deleted file mode 100644 index 193dd29be..000000000 --- a/docker/rocm/migraphx/onnx/parse_clip.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_clip : op_parser -{ - std::vector operators() const { return {{"Clip"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - instruction_ref min_arg; - instruction_ref max_arg; - bool min_used = false; - bool max_used = false; - - if(args.size() == 3 and args[2]->name() != "undefined") - { - max_arg = args[2]; - max_used = true; - } - - if(args.size() >= 2 and args[1]->name() != "undefined") - { - min_arg = args[1]; - min_used = true; - } - // if using previous opset for attributes - else if(contains(info.attributes, "min") and contains(info.attributes, "max")) - { - - float min_val = parser.parse_value(info.attributes.at("min")).at(); - float max_val = parser.parse_value(info.attributes.at("max")).at(); - min_arg = info.add_literal(min_val); - max_arg = info.add_literal(max_val); - min_used = true; - max_used = true; - } - - if(min_used and max_used) - { - return info.add_common_op("clip", args[0], min_arg, max_arg); - } - else if(max_used) - { - return info.add_broadcastable_binary_op("min", args[0], max_arg); - } - else if(min_used) - { - return info.add_broadcastable_binary_op("max", args[0], min_arg); - } - else - { - return info.add_instruction(make_op("identity"), args[0]); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_compare_op.cpp b/docker/rocm/migraphx/onnx/parse_compare_op.cpp deleted file mode 100644 index 1b97e8b29..000000000 --- a/docker/rocm/migraphx/onnx/parse_compare_op.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_compare_op : op_parser -{ - std::vector operators() const - { - return {{"Equal", "equal"}, {"Greater", "greater"}, {"Less", "less"}}; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto l = info.add_broadcastable_binary_op(opd.op_name, args[0], args[1]); - if(l->get_shape().type() != shape::bool_type) - { - l = info.add_instruction(make_op("convert", {{"target_type", shape::bool_type}}), l); - } - return l; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_constant.cpp b/docker/rocm/migraphx/onnx/parse_constant.cpp deleted file mode 100644 index 1396cd4ca..000000000 --- a/docker/rocm/migraphx/onnx/parse_constant.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_constant : op_parser -{ - std::vector operators() const { return {{"Constant"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - const std::vector& /*args*/) const - { - static const std::vector attributes = { - "value", "value_float", "value_floats", "value_int", "value_ints"}; - - std::vector present_attributes; - std::copy_if(attributes.begin(), - attributes.end(), - std::back_inserter(present_attributes), - [&](const std::string& a) { return contains(info.attributes, a); }); - - if(present_attributes.empty()) - { - MIGRAPHX_THROW("Constant node does not contain any supported attribute"); - } - - if(present_attributes.size() > 1) - { - MIGRAPHX_THROW("Constant contains multiple attributes: " + - join_strings(std::move(present_attributes), ", ")); - } - - // cppcheck-suppress accessMoved - auto&& attr = info.attributes[present_attributes[0]]; - literal v = parser.parse_value(attr); - - // return empty literal - if(v.get_shape().elements() == 0) - { - return info.add_literal(literal{v.get_shape().type()}); - } - - // if dim_size is 0, it is a scalar - if(attr.has_t() and attr.t().dims_size() == 0) - { - migraphx::shape scalar_shape{v.get_shape().type()}; - return info.add_literal(migraphx::literal{scalar_shape, v.data()}); - } - - return info.add_literal(v); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_constant_fill.cpp b/docker/rocm/migraphx/onnx/parse_constant_fill.cpp deleted file mode 100644 index 5aef598ed..000000000 --- a/docker/rocm/migraphx/onnx/parse_constant_fill.cpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// Use a literal instruction to replace the constantFill operator. In RNN, input shape -// and value are fixed, so no need to do the actual computation for the constantFill -// operator -struct parse_constant_fill : op_parser -{ - std::vector operators() const { return {{"ConstantFill"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - int input_as_shape = 0; - int dtype = 1; - float value = 0.0f; - - if(contains(info.attributes, "dtype")) - { - dtype = parser.parse_value(info.attributes.at("dtype")).at(); - } - shape::type_t type = get_type(dtype); - - if(contains(info.attributes, "input_as_shape")) - { - input_as_shape = parser.parse_value(info.attributes.at("input_as_shape")).at(); - } - - if(contains(info.attributes, "value")) - { - value = parser.parse_value(info.attributes.at("value")).at(); - } - - if(contains(info.attributes, "extra_shape")) - { - MIGRAPHX_THROW("ConstantFill: cannot handle extra shape attribute"); - } - - if(input_as_shape == 1) - { - if(args.size() != 1) - { - MIGRAPHX_THROW("ConstantFill: need an input argument as output shape"); - } - - if(contains(info.attributes, "shape")) - { - MIGRAPHX_THROW("ConstantFill: cannot set the shape argument and pass in an input " - "at the same time"); - } - - migraphx::argument in = args[0]->eval(); - check_arg_empty(in, "ConstantFill: dynamic shape is not supported"); - - std::vector dims; - in.visit([&](auto input) { dims.assign(input.begin(), input.end()); }); - migraphx::shape s(type, dims); - std::vector values(s.elements(), value); - return info.add_literal(migraphx::literal(s, values)); - } - else if(input_as_shape == 0) - { - if(not contains(info.attributes, "shape")) - { - MIGRAPHX_THROW("ConstantFill: attribute output shape is needed"); - } - - literal ls = parser.parse_value(info.attributes.at("shape")); - std::vector dims; - ls.visit([&](auto s) { dims.assign(s.begin(), s.end()); }); - migraphx::shape s{type, dims}; - std::vector values(s.elements(), value); - return info.add_literal(migraphx::literal(s, values)); - } - else - { - MIGRAPHX_THROW("ConstantFill: wrong value of attribute input_as_shape"); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_constant_of_shape.cpp b/docker/rocm/migraphx/onnx/parse_constant_of_shape.cpp deleted file mode 100644 index 1038a11ae..000000000 --- a/docker/rocm/migraphx/onnx/parse_constant_of_shape.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_constant_of_shape : op_parser -{ - std::vector operators() const { return {{"ConstantOfShape"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - literal l_val{}; - if(contains(info.attributes, "value")) - { - l_val = parser.parse_value(info.attributes.at("value")); - if(l_val.get_shape().elements() != 1) - { - MIGRAPHX_THROW("ConstantOfShape: attribute value can contain only 1 elements!"); - } - // convert to a scalar literal - l_val = literal(shape{l_val.get_shape().type(), {1}, {0}}, l_val.data()); - } - else - { - l_val = literal({shape::float_type, {1}, {0}}, {0.0f}); - } - - if(args.empty()) - { - MIGRAPHX_THROW("ConstantOfShape : must have 1 input!"); - } - else - { - migraphx::shape s; - // input is empty, output is a scalar - auto type = l_val.get_shape().type(); - migraphx::argument input = args[0]->eval(); - if(not input.empty()) - { - // empty input tensor, output is a scalar - if(args[0]->get_shape().elements() == 0) - { - s = migraphx::shape{type, {1}, {0}}; - } - else - { - std::vector dims; - input.visit([&](auto ia) { dims.assign(ia.begin(), ia.end()); }); - s = migraphx::shape{type, dims}; - } - literal l_out{}; - l_val.visit([&](auto val) { - using val_type = std::remove_cv_t; - // l_val contains only one element - std::vector out_vec(s.elements(), val.front()); - l_out = literal(s, out_vec); - }); - return info.add_literal(l_out); - } - // has variable input (dynamic shape buffer) - else - { - auto dv_lit = info.add_literal(l_val); - auto alloc_ins = - info.add_instruction(make_op("allocate", {{"buf_type", type}}), args[0]); - return info.add_instruction(make_op("fill"), dv_lit, alloc_ins); - } - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_conv_transpose.cpp b/docker/rocm/migraphx/onnx/parse_conv_transpose.cpp deleted file mode 100644 index 309b6e84c..000000000 --- a/docker/rocm/migraphx/onnx/parse_conv_transpose.cpp +++ /dev/null @@ -1,193 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -template -std::vector to_int64_vector(const std::vector& input_vector) -{ - std::vector output_vector(input_vector.begin(), input_vector.end()); - return output_vector; -} - -struct parse_conv_transpose : op_parser -{ - std::vector operators() const { return {{"ConvTranspose"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - operation op = make_op("convolution_backwards"); - value values = op.to_value(); - auto l0 = args[0]; - std::vector padding; - bool asym_padding = false; - assert(l0->get_shape().ndim() > 2); - auto kdims = l0->get_shape().ndim() - 2; - - // ensure pads available only when auto_pad is "NOT_SET" - check_padding_mode(info, opd.onnx_name); - - if(contains(info.attributes, "pads")) - { - copy(info.attributes["pads"].ints(), std::back_inserter(padding)); - - asym_padding = is_asym_padding(padding); - - size_t pad_ndims = padding.size() / 2; - if(not asym_padding) - { - check_attr_sizes(kdims, pad_ndims, "PARSE_CONV_TRANSPOSE: inconsistent paddings"); - values["padding"].clear(); - std::transform(padding.begin(), - padding.begin() + pad_ndims, - std::back_inserter(values["padding"]), - [](auto pad_val) { return pad_val; }); - } - else if(l0->get_shape().dynamic()) - { - MIGRAPHX_THROW("PARSE_CONV_TRANSPOSE: asymmetric padding (padding_L != padding_R) " - "not supported with dynamic shapes"); - } - else - { - // set padding to 0s, asym_padding handled by parser with slice - // TODO changing parser and op to do asym padding in op - values["padding"] = std::vector(pad_ndims, 0); - } - } - - if(contains(info.attributes, "strides")) - { - values["stride"].clear(); - copy(info.attributes["strides"].ints(), std::back_inserter(values["stride"])); - check_attr_sizes( - kdims, values["stride"].size(), "PARSE_CONV_TRANSPOSE: inconsistent strides"); - } - - if(contains(info.attributes, "dilations")) - { - values["dilation"].clear(); - copy(info.attributes["dilations"].ints(), std::back_inserter(values["dilation"])); - check_attr_sizes( - kdims, values["dilation"].size(), "PARSE_CONV_TRANSPOSE: inconsistent dilations"); - } - - // TODO: auto padding needs to be implemented for this parser and operator - if(contains(info.attributes, "auto_pad") and - to_upper(info.attributes.at("auto_pad").s()) != "NOTSET") - { - MIGRAPHX_THROW("PARSE_CONV_TRANSPOSE: auto padding not supported"); - } - - if(contains(info.attributes, "group")) - { - values["group"] = parser.parse_value(info.attributes.at("group")).at(); - } - - recalc_conv_attributes(values, kdims); - - op.from_value(values); - auto l1 = info.add_instruction(op, l0, args[1]); - if(asym_padding) - { - std::vector dims = to_int64_vector(l1->get_shape().lens()); - std::vector curr_shape(dims.begin() + 2, dims.end()); - std::vector axes(kdims); - std::iota(axes.begin(), axes.end(), 2); // ignore first 2 dims - - auto pad_kdim_start = padding.begin() + kdims; - std::vector starts(padding.begin(), pad_kdim_start); - - std::vector ends{}; - std::transform(curr_shape.begin(), - curr_shape.end(), - pad_kdim_start, - std::back_inserter(ends), - [](auto curr_dim, auto pad_dim) { return curr_dim - pad_dim; }); - - l1 = info.add_instruction( - make_op("slice", {{"axes", axes}, {"starts", starts}, {"ends", ends}}), l1); - } - - // TODO, should check output_padding < (strides or dilations) - if(contains(info.attributes, "output_padding") and - not contains(info.attributes, "output_shape")) - { - size_t non_kdims = l1->get_shape().ndim() * 2 - kdims; - std::vector output_padding(non_kdims, 0); - copy(info.attributes["output_padding"].ints(), std::back_inserter(output_padding)); - check_attr_sizes(kdims, - output_padding.size() - non_kdims, - "PARSE_CONV_TRANSPOSE: inconsistent output padding"); - l1 = info.add_instruction(make_op("pad", {{"pads", output_padding}}), l1); - } - - // TODO, doing unnecessary calcuations with this. Could instead - // calculate the padding to conv_transpose that would give the output_shape. - if(contains(info.attributes, "output_shape")) - { - if(l1->get_shape().dynamic()) - { - MIGRAPHX_THROW("PARSE_CONV_TRANSPOSE: output_shape attribute and dynamic shapes " - "not supported"); - } - std::vector dims = to_int64_vector(l1->get_shape().lens()); - std::vector curr_shape(dims.begin() + 2, dims.end()); - std::vector output_shape; - copy(info.attributes["output_shape"].ints(), std::back_inserter(output_shape)); - check_attr_sizes( - kdims, output_shape.size(), "PARSE_CONV_TRANSPOSE: inconsistent output shape"); - if(curr_shape != output_shape) - { - std::vector target_padding(dims.size() * 2 - kdims, 0); - std::transform(output_shape.begin(), - output_shape.end(), - curr_shape.begin(), - std::back_inserter(target_padding), - [](auto out_dim, auto curr_dim) { return out_dim - curr_dim; }); - l1 = info.add_instruction(make_op("pad", {{"pads", target_padding}}), l1); - } - } - - return info.add_bias(args, l1, 1); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_convolution.cpp b/docker/rocm/migraphx/onnx/parse_convolution.cpp deleted file mode 100644 index 190c69a0f..000000000 --- a/docker/rocm/migraphx/onnx/parse_convolution.cpp +++ /dev/null @@ -1,371 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_convolution : op_parser -{ - std::vector operators() const - { - return {{"Conv", "convolution"}, {"ConvInteger", "quant_convolution"}}; - } - - // Convert to half prior to a shift to ensure we preserve accuracy here then - // convert back to int8 - static instruction_ref add_int8_shift(const onnx_parser::node_info& info, - const instruction_ref& offset_op, - instruction_ref& unshifted_input) - { - auto unshifted_input_half = info.add_instruction( - migraphx::make_op("convert", {{"target_type", migraphx::shape::half_type}}), - unshifted_input); - - auto input_shifted_half = info.add_common_op("add", unshifted_input_half, offset_op); - - return info.add_instruction( - migraphx::make_op("convert", {{"target_type", migraphx::shape::int8_type}}), - input_shifted_half); - } - - static void shift_input_and_bias(const onnx_parser::node_info& info, - const instruction_ref& offset_op, - const bool has_bias, - instruction_ref& input, - instruction_ref& input_bias) - { - input = add_int8_shift(info, offset_op, input); - if(has_bias) - { - input_bias = add_int8_shift(info, offset_op, input_bias); - } - } - - static float get_symmetric_value(const instruction_ref& input) - { - float symmetric_value = 0; - // adjust symmetric zero point value for uint8 types - if(input->get_shape().type() == migraphx::shape::uint8_type) - { - symmetric_value = 128; - } - return symmetric_value; - } - - static instruction_ref gen_symmetric_literal(const instruction_ref& input, - const bool is_quant_conv, - onnx_parser::node_info& info) - { - instruction_ref ret = input; - if(is_quant_conv) - { - float symmetric_value = get_symmetric_value(input); - ret = info.add_literal(migraphx::literal{ - migraphx::shape{input->get_shape().type(), {1}, {0}}, {symmetric_value}}); - } - - return ret; - } - - static instruction_ref get_zero_point(const instruction_ref& input, - int index, - const bool is_quant_conv, - onnx_parser::node_info& info, - const std::vector& args) - { - instruction_ref ret = input; - if(args.size() > index) - { - // Check for type mismatch on parse - if(input->get_shape().type() != args[index]->get_shape().type()) - MIGRAPHX_THROW("PARSE:Conv Data and Data Zero Point must have same type"); - - ret = args[index]; - if(is_symmetric_zero_point(ret)) - { - ret = gen_symmetric_literal(ret, is_quant_conv, info); - } - } - else - { - ret = gen_symmetric_literal(ret, is_quant_conv, info); - } - - return ret; - } - - static bool is_symmetric_zero_point(instruction_ref zp) - { - if(not zp->can_eval()) - return false; - - float symmetric_value = get_symmetric_value(zp); - - bool all_zeros = false; - zp->eval().visit([&](auto z) { - all_zeros = std::all_of( - z.begin(), z.end(), [&](auto val) { return float_equal(val, symmetric_value); }); - }); - return all_zeros; - } - - static migraphx::operation - qparam_broadcast_op(instruction_ref qparam, std::vector lens, std::size_t axis) - { - if(qparam->get_shape().elements() == 1) - { - return migraphx::make_op("multibroadcast", {{"out_lens", lens}}); - } - return migraphx::make_op("broadcast", {{"out_lens", lens}, {"axis", axis}}); - } - - static instruction_ref handle_quant_bias(const operation& op, - const instruction_ref& input, - const instruction_ref& x, - const instruction_ref& weights, - const instruction_ref& x_zp, - const instruction_ref& w_zp, - onnx_parser::node_info& info) - { - // to handle the bias, apply the following transformation: - // conv(x-x_zp,w-w_zp) = conv(x,w) - conv(x_zp,w) - conv(x,w_zp) + conv(x_zp,w_zp) - instruction_ref ret = input; - - // multibroadcast (or broadcast) zero points according to spec - // x_zp should be a scalar or literal with one element - // w_zp can be either a single element or a 1d tensor with size out_channels - migraphx::operation x_zp_bc = - migraphx::make_op("multibroadcast", {{"out_lens", x->get_shape().lens()}}); - migraphx::operation w_zp_bc = qparam_broadcast_op(w_zp, weights->get_shape().lens(), 0); - - if(not is_symmetric_zero_point(x_zp)) - { - auto x_zp_mb = info.add_instruction(x_zp_bc, x_zp); - auto out_zp_1 = info.add_instruction(op, x_zp_mb, weights); - ret = info.add_common_op("sub", ret, out_zp_1); - } - - if(not is_symmetric_zero_point(w_zp)) - { - auto w_zp_mb = info.add_instruction(w_zp_bc, w_zp); - auto out_zp_2 = info.add_instruction(op, x, w_zp_mb); - ret = info.add_common_op("sub", ret, out_zp_2); - } - - if(not(is_symmetric_zero_point(x_zp)) and not(is_symmetric_zero_point(w_zp))) - { - auto x_zp_mb = info.add_instruction(x_zp_bc, x_zp); - auto w_zp_mb = info.add_instruction(w_zp_bc, w_zp); - - auto out_zp_3 = info.add_instruction(op, x_zp_mb, w_zp_mb); - - ret = info.add_common_op("add", ret, out_zp_3); - } - return ret; - } - - static void handle_quant_inputs(const bool is_quant_conv, - instruction_ref& input, - instruction_ref& weights, - instruction_ref& input_zp, - instruction_ref& weight_zp, - onnx_parser::node_info& info) - { - if(not is_quant_conv) - return; - - auto input_type = input->get_shape().type(); - auto weight_type = weights->get_shape().type(); - - // Handle uint8 bias and input shifts - instruction_ref offset_op; - if(((input_type == migraphx::shape::uint8_type) or - (weight_type == migraphx::shape::uint8_type))) - { - offset_op = info.add_literal( - migraphx::literal{migraphx::shape{migraphx::shape::half_type}, {-128}}); - } - - if(input_type == migraphx::shape::uint8_type) - { - shift_input_and_bias( - info, offset_op, (not is_symmetric_zero_point(input_zp)), input, input_zp); - } - - if(weight_type == migraphx::shape::uint8_type) - { - shift_input_and_bias( - info, offset_op, (not is_symmetric_zero_point(weight_zp)), weights, weight_zp); - } - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - auto op = make_op(opd.op_name); - auto values = op.to_value(); - auto x = args[0]; - auto weights = args[1]; - auto x_shape = x->get_shape(); - auto w_shape = weights->get_shape(); - auto in_lens = x_shape.max_lens(); - assert(in_lens.size() > 2); - auto kdims = in_lens.size() - 2; - - // ensure pads available only when auto_pad is "NOT_SET" - check_padding_mode(info, opd.onnx_name); - - if(contains(info.attributes, "strides")) - { - values["stride"].clear(); - copy(info.attributes["strides"].ints(), std::back_inserter(values["stride"])); - check_attr_sizes(kdims, values["stride"].size(), "PARSE_CONV: inconsistent strides"); - } - if(contains(info.attributes, "dilations")) - { - values["dilation"].clear(); - copy(info.attributes["dilations"].ints(), std::back_inserter(values["dilation"])); - check_attr_sizes( - kdims, values["dilation"].size(), "PARSE_CONV: inconsistent dilations"); - } - - std::vector padding; - if(contains(info.attributes, "pads")) - { - values["padding"].clear(); - copy(info.attributes["pads"].ints(), std::back_inserter(padding)); - check_attr_sizes(kdims, padding.size() / 2, "PARSE_CONV: inconsistent paddings"); - } - if(contains(info.attributes, "auto_pad")) - { - bool is_same_padding = false; - auto auto_pad = info.attributes["auto_pad"].s(); - if(auto_pad.find("SAME") != std::string::npos) - { - is_same_padding = true; - } - - // check if image shape is dynamic - bool image_shape_dynamic = false; - if(x_shape.dynamic()) - { - auto dyn_dims = x_shape.dyn_dims(); - std::for_each(dyn_dims.begin() + 2, dyn_dims.end(), [&](auto dyn_dim) { - if(not dyn_dim.is_fixed()) - { - image_shape_dynamic = true; - } - }); - } - - // check if kernel shape is dynamic - bool kernel_shape_dynamic = false; - if(w_shape.dynamic()) - { - auto dyn_dims = w_shape.dyn_dims(); - std::for_each(dyn_dims.begin() + 2, dyn_dims.end(), [&](auto dyn_dim) { - if(not dyn_dim.is_fixed()) - { - kernel_shape_dynamic = true; - } - }); - } - - if(is_same_padding) - { - if(image_shape_dynamic or kernel_shape_dynamic) - { - // must calculate "same" padding with input shape data - bool is_same_upper = (auto_pad.find("SAME_UPPER") != std::string::npos); - values["padding_mode"] = is_same_upper - ? to_value(op::padding_mode_t::same_upper) - : to_value(op::padding_mode_t::same_lower); - } - else - { - // kernel shape will be fixed, so max_lens() == min_len() for kernel lengths - auto weight_lens = weights->get_shape().max_lens(); - std::vector k_lens(weight_lens.begin() + 2, weight_lens.end()); - cal_auto_padding_size(info, - values, - k_lens, - values["dilation"].to_vector(), - in_lens, - padding); - } - } - } - values["padding"] = std::vector(padding.begin(), padding.end()); - - if(contains(info.attributes, "group")) - { - values["group"] = parser.parse_value(info.attributes.at("group")).at(); - } - - recalc_conv_attributes(values, kdims); - - instruction_ref ret; - // parse a_zero_point and b_zero_point values - auto is_quant_conv = opd.op_name == "quant_convolution"; - - auto x_zp = get_zero_point(x, 2, is_quant_conv, info, args); - auto w_zp = get_zero_point(weights, 3, is_quant_conv, info, args); - - op.from_value(values); - - handle_quant_inputs(is_quant_conv, x, weights, x_zp, w_zp, info); - - ret = info.add_instruction(op, x, weights); - - // Handle quant_conv residuals between input/weights to avoid overflow - if(is_quant_conv) - { - ret = handle_quant_bias(op, ret, x, weights, x_zp, w_zp, info); - } - else - { - // Handle Convolution case with bias to output - ret = info.add_bias(args, ret, 1); - } - - return ret; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_depthtospace.cpp b/docker/rocm/migraphx/onnx/parse_depthtospace.cpp deleted file mode 100644 index b70aca84b..000000000 --- a/docker/rocm/migraphx/onnx/parse_depthtospace.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_depthtospace : op_parser -{ - std::vector operators() const { return {{"DepthToSpace"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto s = args[0]->get_shape(); - // mode attribute of DepthToSpace - auto mode = std::string("DCR"); - if(contains(info.attributes, "mode")) - { - mode = info.attributes.at("mode").s(); // DCR or CRD? - } - // blocksize attribute of DepthToSpace - int blocksize = 0; - if(contains(info.attributes, "blocksize")) - { - blocksize = info.attributes.at("blocksize").i(); - } - if(blocksize < 1) - { - MIGRAPHX_THROW("DepthToSpace: blocksize is less than 1"); - } - // calculate dimensions - auto lens1 = s.lens(); - auto lens2 = s.lens(); - unsigned long divisor = std::pow(blocksize, 2); - if((lens2[1] % divisor) == 0) - lens2[1] = lens2[1] / divisor; - else - MIGRAPHX_THROW("DepthToSpace: div by blocksize quotient not int "); - lens1.push_back(lens1[2]); - lens1.push_back(lens1[3]); - lens2[2] = lens2[2] * blocksize; - lens2[3] = lens2[3] * blocksize; - lens1[2] = blocksize; - std::vector perm; - if(mode == "DCR") - { - lens1[3] = lens1[1] / divisor; - lens1[1] = blocksize; - perm = {0, 3, 4, 1, 5, 2}; - } - else if(mode == "CRD") - { - lens1[1] = lens1[1] / divisor; - lens1[3] = blocksize; - perm = {0, 1, 4, 2, 5, 3}; - } - else - MIGRAPHX_THROW("DepthToSpace: mode attribute cannot be read."); - - auto temp1 = info.add_instruction(make_op("reshape", {{"dims", lens1}}), args[0]); - auto temp2 = info.add_instruction(make_op("transpose", {{"permutation", perm}}), temp1); - return info.add_instruction(make_op("reshape", {{"dims", lens2}}), temp2); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_dequantizelinear.cpp b/docker/rocm/migraphx/onnx/parse_dequantizelinear.cpp deleted file mode 100644 index 949198671..000000000 --- a/docker/rocm/migraphx/onnx/parse_dequantizelinear.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_dequantizelinear : op_parser -{ - std::vector operators() const { return {{"DequantizeLinear"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - if(args.size() < 2 or args.size() > 3) - { - MIGRAPHX_THROW("DequantizeLinear: must have either 2 or 3 inputs, " + - std::to_string(args.size()) + " inputs provided"); - } - - if(args.size() == 3) - { - if(args[0]->get_shape().type() != args[2]->get_shape().type()) - MIGRAPHX_THROW("DequantizeLinear: x and y_zero_point must be of same type"); - - if(args[1]->get_shape().lens() != args[2]->get_shape().lens()) - { - MIGRAPHX_THROW("DequantizeLinear: y_scale and y_zero_point shape mismatch. " - "Provided y_scale " - "shape: " + - to_string_range(args[1]->get_shape().lens()) + - ", provided y_zero_point shape: " + - to_string_range(args[2]->get_shape().lens())); - } - } - - int axis = 1; - if(contains(info.attributes, "axis")) - axis = info.attributes.at("axis").i(); - - int block_size = 0; - if(contains(info.attributes, "block_size")) - block_size = info.attributes.at("block_size").i(); - - args = transform_quantize_dequantize_linear_inputs( - info, opd.onnx_name, block_size, axis, args); - - return info.add_instruction(make_op("dequantizelinear"), args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_dropout.cpp b/docker/rocm/migraphx/onnx/parse_dropout.cpp deleted file mode 100644 index 7eba79ba7..000000000 --- a/docker/rocm/migraphx/onnx/parse_dropout.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_dropout : op_parser -{ - std::vector operators() const { return {{"Dropout"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto out = info.add_instruction(make_op("identity"), args[0]); - auto s = args[0]->get_shape(); - std::vector vec(s.elements(), 1); - shape mask_s{shape::bool_type, s.lens()}; - auto mask = info.add_literal(literal(mask_s, vec)); - - return {out, mask}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_dynamicquantizelinear.cpp b/docker/rocm/migraphx/onnx/parse_dynamicquantizelinear.cpp deleted file mode 100644 index 8041d09eb..000000000 --- a/docker/rocm/migraphx/onnx/parse_dynamicquantizelinear.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - ********************************************************************************* - * Reference: see DynamicQuantizeLinear in * - * https://github.com/onnx/onnx/blob/main/docs/Operators.md * - ********************************************************************************* -DynamicQuantizeLinear -A Function to fuse calculation for Scale, Zero Point and FP32->8Bit conversion of FP32 Input data. -Outputs Scale, ZeroPoint and Quantized Input for a given FP32 Input. Scale is calculated as: -y_scale = (maximum(0, max(x)) - minimum(0, min(x))) / (qmax - qmin) -* where qmax and qmin are max and min values for quantization range i.e. [0, 255] in case of uint8 -* data range is adjusted to include 0. - -Zero point is calculated as: -intermediate_zero_point = qmin - min(x)/y_scale -y_zero_point = cast(round(saturate(itermediate_zero_point))) -* where qmax and qmin are max and min values for quantization range .i.e [0, 255] in case of uint8 -* for saturation, it saturates to [0, 255] if it's uint8, or [-127, 127] if it's int8. Right now -only uint8 is supported. -* rounding to nearest ties to even. Data quantization formula is: - -y = saturate (round (x / y_scale) + y_zero_point) -* for saturation, it saturates to [0, 255] if it's uint8, or [-127, 127] if it's int8.Right now only -uint8 is supported. -* rounding to nearest ties to even. - -Version -This version of the operator has been available since version 11 of the default ONNX operator set. - -Inputs -x : T1 -Input tensor - -Outputs -y : T2 -Quantized output tensor - -y_scale : tensor(float) -Output scale. It's a scalar, which means a per-tensor/layer quantization. - -y_zero_point : T2 -Output zero point. It's a scalar, which means a per-tensor/layer quantization. - -Type Constraints -T1 : tensor(float) -Constrain 'x' to float tensor. - -T2 : tensor(uint8) -Constrain 'y_zero_point' and 'y' to 8-bit unsigned integer tensor. -*/ - -struct parse_dynamicquantizelinear : op_parser -{ - std::vector operators() const { return {{"DynamicQuantizeLinear"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - const std::vector& args) const - { - auto x = args[0]; - auto x_shape = x->get_shape(); - auto x_type = x_shape.type(); - if(x_shape.dynamic()) - MIGRAPHX_THROW("DYNAMICQUANTIZELINEAR: dynamic shapes are not supported"); - - auto lit_0 = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {0}}); - - // 1. Computing y_scale - // Note: currently, DynamicQuantizeLinear only has uint8 quantization: - const auto type_max = std::numeric_limits::max(); - const auto type_min = std::numeric_limits::min(); - std::vector axes(x_shape.lens().size()); - std::iota(axes.begin(), axes.end(), 0); - - // maximum(0, max(x)) - auto reduce_max_x = - info.add_instruction(migraphx::make_op("reduce_max", {{"axes", axes}}), x); - auto max_x = info.add_common_op("max", lit_0, reduce_max_x); - - // minimum(0, min(x)) - auto reduce_min_x = - info.add_instruction(migraphx::make_op("reduce_min", {{"axes", axes}}), x); - auto min_x = info.add_common_op("min", lit_0, reduce_min_x); - - auto q_range = info.add_literal(migraphx::literal{ - migraphx::shape{x_type, max_x->get_shape().lens()}, {type_max - type_min}}); - auto q_min = info.add_literal( - migraphx::literal{migraphx::shape{x_type, max_x->get_shape().lens()}, {type_min}}); - auto q_max = info.add_literal( - migraphx::literal{migraphx::shape{x_type, max_x->get_shape().lens()}, {type_max}}); - - // y_scale = (maximum(0, max(x)) - minimum(0, min(x))) / (qmax - qmin) - auto sub0 = info.add_common_op("sub", max_x, min_x); - auto y_scale = info.add_common_op("div", sub0, q_range); - - // 2. Computing y_zero_point - // intermediate_zero_point = qmin - min(x) / y_scale - auto div1 = info.add_common_op("div", min_x, y_scale); - auto interm_zp = info.add_common_op("sub", q_min, div1); - - // y_zero_point = cast(round(saturate(itermediate_zero_point))) - auto saturate = info.add_instruction(migraphx::make_op("clip"), interm_zp, q_min, q_max); - auto round = info.add_instruction(migraphx::make_op("nearbyint"), saturate); - auto y_zero_point = info.add_instruction( - migraphx::make_op("convert", {{"target_type", migraphx::shape::uint8_type}}), round); - - // 3. quantize x with y_scale and y_zero_point - auto quant = bcast_qdq_instr("quantizelinear", x, y_scale, y_zero_point, info); - - return {quant, y_scale, y_zero_point}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_einsum.cpp b/docker/rocm/migraphx/onnx/parse_einsum.cpp deleted file mode 100644 index 9743289b0..000000000 --- a/docker/rocm/migraphx/onnx/parse_einsum.cpp +++ /dev/null @@ -1,772 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_einsum : op_parser -{ - using int_mat = std::vector>; - - struct equation_info - { - bool explicit_form = false; - std::vector input_terms; - std::string output_term; - std::map label_count; - std::vector>> duplicates; - size_t ellipsis_ndim = 0; - }; - - std::vector operators() const { return {{"Einsum"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser&, - const onnx_parser::node_info& info, - const std::vector& args) const - { - if(not contains(info.attributes, "equation")) - MIGRAPHX_THROW("Equation attribute is required"); - std::string equation = info.attributes.at("equation").s(); - - const equation_info eq_info = analyze_equation(equation, args); - - auto terms = eq_info.input_terms; - terms.push_back(eq_info.output_term); - const auto map_mat = make_mapping_matrix(terms, eq_info.label_count, eq_info.ellipsis_ndim); - - // Holds the mapping matrix representations of the two terms being processed - // cur_pair[0] acts as the accumulator for previously processed inputs - // cur_pair[1] holds the representation for the current input - // As operations are added to the einsum graph, cur_pair gets manipulated - int_mat cur_pair = make_matrix(2, map_mat[0].size(), -1); - - instruction_ref cur_op; - std::optional last_op; - // Perform a left fold on the inputs - for(auto arg_idx = 0; arg_idx < args.size(); ++arg_idx) - { - cur_op = args[arg_idx]; - cur_pair[1] = map_mat[arg_idx]; - - cur_op = preprocess_input( - info, cur_op, eq_info.duplicates[arg_idx], map_mat, arg_idx, cur_pair); - - if(last_op) - cur_op = process_pair(info, *last_op, cur_op, map_mat, arg_idx, cur_pair); - - last_op = cur_op; - cur_pair[0] = cur_pair[1]; - } - - return finalize_output(info, cur_op, map_mat, cur_pair); - } - - // Equation Parsing - - equation_info analyze_equation(std::string_view equation, - const std::vector& args) const - { - equation_info eq_info = parse_equation(equation); - - eq_info.ellipsis_ndim = validate_input_terms(eq_info.input_terms, args); - if(not eq_info.output_term.empty()) - validate_output_term(eq_info.output_term, eq_info.label_count, eq_info.ellipsis_ndim); - else if(not eq_info.explicit_form) - eq_info.output_term = generate_output_term(eq_info.label_count, eq_info.ellipsis_ndim); - - eq_info.duplicates = find_duplicates(eq_info.input_terms); - - return eq_info; - } - - // Equation: Input Output - // Input: Term | Term ',' Input - // Output: '->' TermOpt | epsilon - // TermOpt: Term | epsilon - // Term: Labels | LabelsOpt '...' LabelsOpt - // LabelsOpt: Labels | epsilon - // Labels: [a-zA-Z]+ - equation_info parse_equation(std::string_view equation) const - { - equation_info ret; - - std::vector lexers; - lexers.push_back(lex_while(&isspace)); - lexers.push_back(lex_while(&isalpha)); - lexers.push_back(lex_equal("->")); - lexers.push_back(lex_equal("...")); - lexers.push_back(lex_equal(",")); - - auto tokens = tokenize(equation.data(), equation.data() + equation.length(), lexers); - - std::string term; - bool has_ellipsis = false; - - for(const auto& token : tokens) - { - if(std::isspace(token.front()) != 0) - continue; - - if(std::isalpha(token.front()) != 0) - { - term += token; - if(not ret.explicit_form) - { - for(auto c : token) - ++ret.label_count[c]; - } - } - else if(token == "->") - { - if(ret.explicit_form) - MIGRAPHX_THROW("Einsum equation has multiple '->' symbols"); - - if(term.empty()) - MIGRAPHX_THROW("No term specified before '->' symbol"); - - ret.explicit_form = true; - has_ellipsis = false; - ret.input_terms.push_back(term); - term.clear(); - } - else if(token == "...") - { - if(has_ellipsis) - MIGRAPHX_THROW("Ellipsis can only appear once per einsum equation term"); - - has_ellipsis = true; - term += "*"; - } - else if(token == ",") - { - if(ret.explicit_form) - MIGRAPHX_THROW("Einsum equation can't have a ',' symbol in the output"); - - if(term.empty()) - MIGRAPHX_THROW("No term specified before ',' symbol"); - - has_ellipsis = false; - ret.input_terms.push_back(term); - term.clear(); - } - } - - if(ret.explicit_form) - ret.output_term = term; - else if(not term.empty()) - ret.input_terms.push_back(term); - else - MIGRAPHX_THROW("Last input term is missing"); - - return ret; - } - - size_t validate_input_terms(const std::vector& input_terms, - const std::vector& args) const - { - if(input_terms.size() != args.size()) - MIGRAPHX_THROW("Number of terms in the input equation - " + - std::to_string(input_terms.size()) + - " does not match the number of inputs " + std::to_string(args.size())); - - auto global_ellipsis_dims = 0u; - for(auto i = 0u; i < args.size(); ++i) - { - const auto& term = input_terms[i]; - const auto dims = args[i]->get_shape().lens(); - const auto rank = dims.size(); - - auto current_dim = 0u; - for(const auto l : term) - { - if(l == '*') - { - const auto ellipsis_dims = rank - term.size() + 1; - if(global_ellipsis_dims > 0 and ellipsis_dims != global_ellipsis_dims) - MIGRAPHX_THROW("Every occurrence of ellipsis in the equation must " - "represent the same number of dimensions"); - global_ellipsis_dims = ellipsis_dims; - current_dim += ellipsis_dims; - } - else - ++current_dim; - } - - if(current_dim != rank) - MIGRAPHX_THROW("Number of labels in " + std::to_string(i + 1) + ". input_term (" + - term + ") does not match the rank (" + std::to_string(rank) + - ") of corresponding input"); - } - - return global_ellipsis_dims; - } - - void validate_output_term(std::string_view output_term, - const std::map& label_count, - size_t ellipsis_ndim) const - { - std::string_view::iterator it = - std::find_if(output_term.begin(), output_term.end(), [&](auto l) { - return not contains(label_count, l) and l != '*'; - }); - if(it != output_term.end()) - MIGRAPHX_THROW("Output term contains label " + std::to_string(*it) + - ", which is not present in any of the input terms"); - - if(ellipsis_ndim != 0 and not contains(output_term, "*")) - MIGRAPHX_THROW( - "Output term does not contain ellipsis (...) even though an input term does"); - } - - // Creates output term when the equation is in implicit mode. - // The created output term must contain the alphabetically sorted sequence of labels appearing - // exactly once in the equation. - // If ellipsis are present in the left hand side of the equation, the ellipsis dimensions are - // set to the beginning of the output term. - std::string generate_output_term(const std::map& label_count, - size_t ellipsis_ndim) const - { - std::string output_term = ellipsis_ndim == 0 ? "" : "*"; - output_term = transform_accumulate( - label_count.begin(), label_count.end(), output_term, std::plus<>(), [](const auto& p) { - if(p.second == 1) - return std::string{p.first}; - else - return std::string{}; - }); - - return output_term; - } - - // Creates a matrix representation of the equation. - // - // Rows correspond to equation terms, in order of appearance. - // - // Columns represent the unique labels contained in the equation, ordered alphabetically. If - // ellipses are present in the equation, they are represented by the final N columns(N being the - // number of dimensions covered by and ellipsis). - // Labels not present in a given term are signified by -1. - // Labels present in a given term are signified by the input axis they represent. - // - // e.g. For equation "...ik,kj...->ij...", assuming ... cover two dimensions, the resulting - // matrix is: - // +-------+----+----+----+---+---+ - // | | i | j | k | * | * | - // +-------+----+----+----+---+---+ - // | ...ik | 2 | -1 | 3 | 0 | 1 | - // | kj... | -1 | 1 | 0 | 2 | 3 | - // | ij... | 0 | 1 | -1 | 2 | 3 | - // +-------+----+----+----+---+---+ - int_mat make_mapping_matrix(const std::vector& terms, - const std::map& label_count, - size_t ellipsis_ndim) const - { - std::map label_to_column; - - auto it = label_count.begin(); - for(auto i = 0; i < label_count.size(); ++i) - label_to_column[(it++)->first] = i; - - int_mat map_mat = make_matrix(terms.size(), label_count.size() + ellipsis_ndim, -1); - - for(auto i = 0; i < terms.size(); ++i) - { - const auto& term = terms[i]; - int col_id = 0; - for(const auto l : term) - { - if(l == '*') - { - std::iota(map_mat[i].end() - ellipsis_ndim, map_mat[i].end(), col_id); - col_id += ellipsis_ndim; - } - else - map_mat[i][label_to_column[l]] = col_id++; - } - } - - return map_mat; - } - - // Finds the duplicated labels in each of the terms and stores the axes on which they occur. - // - // e.g. For equation "iikjj,jkj", the result is a vector containing the two following maps: - // result[0]: {'i': [0, 1], 'j': [3, 4]} - // result[1]: {'j': [0, 2]} - std::vector>> - find_duplicates(const std::vector& terms) const - { - std::vector>> duplicates; - for(const auto& term : terms) - { - std::map> duplicate_axes; - for(auto i = 0; i < term.size(); ++i) - duplicate_axes[term[i]].push_back(i); - - erase_if(duplicate_axes, [](const auto& p) { return p.second.size() < 2; }); - duplicates.push_back(duplicate_axes); - } - - return duplicates; - } - - // Graph Building - - instruction_ref preprocess_input(const onnx_parser::node_info& info, - instruction_ref op, - const std::map>& duplicates, - const int_mat& map_mat, - size_t input_idx, - int_mat& cur_pair) const - { - if(not duplicates.empty()) - { - std::vector> diag; - diag.reserve(duplicates.size()); - std::transform(duplicates.begin(), - duplicates.end(), - std::back_inserter(diag), - [](const auto& d) { return d.second; }); - - op = gather_diagonal(info, cur_pair, op, diag); - } - - // Unsqueeze the input shape in the dimensions marked as -1 in the mapping_matrix - // Transpose the input shape so the labels are in alphabetical order - op = transpose_unsqueeze(info, cur_pair, op); - - std::vector red; - // Check if a given label appears in any of the subsequent mapping matrix terms(this - // includes the output). If does not, it is reduced and marked as -1 in cur_pair. - for(int d = 0; d < map_mat[0].size(); ++d) - { - bool all_neg_one = all_of(extract_column(map_mat, d, input_idx + 1, map_mat.size()), - [](auto i) { return i == -1; }); - if(all_neg_one and cur_pair[1][d] != -1 and cur_pair[0][d] == -1) - red.push_back(d); - } - - return apply_reduce_sum_op(info, op, red, cur_pair[1]); - } - - instruction_ref gather_diagonal(const onnx_parser::node_info& info, - int_mat& cur_pair, - instruction_ref op, - const int_mat& diag) const - { - if(diag.size() != 1) - MIGRAPHX_THROW( - "Parsing of equations with more than one duplicated labels per input term is not " - "implemented"); - - const auto& op_lens = op->get_shape().lens(); - - int first_axis = diag[0][0]; - const std::vector& axes = diag[0]; - if(not all_of(axes, [&](int a) { return op_lens[first_axis] == op_lens[a]; })) - MIGRAPHX_THROW("All duplicate labels have to be the same dimension"); - - std::vector batch_axes = set_difference(arange(0, op_lens.size()), axes); - if(not all_of(batch_axes, [&](int ba) { return ba < axes.front(); })) - MIGRAPHX_THROW( - "Parsing of equations with duplicated labels and batch axes that are not " - "the outer-most axes, is not implemented"); - - size_t batch_size = calc_dim(batch_axes, op_lens); - - std::vector indices; - for(size_t batch = 0; batch < batch_size; ++batch) - { - for(size_t i = 0; i < op_lens[first_axis]; ++i) - { - std::vector index(axes.size(), i); - indices.insert(indices.end(), index.begin(), index.end()); - } - } - - std::vector indices_lens{op_lens[first_axis], axes.size()}; - if(batch_size > 1) - indices_lens.insert(indices_lens.begin(), batch_size); - - auto indices_arg = info.add_literal( - migraphx::literal{migraphx::shape{migraphx::shape::int64_type, indices_lens}, indices}); - - op = info.add_instruction( - migraphx::make_op("gathernd", {{"batch_dims", batch_axes.size()}}), op, indices_arg); - - // compute output row - std::replace_if( - cur_pair[1].begin(), - cur_pair[1].end(), - [&](auto r) { return contains(axes, r); }, - first_axis); - - for(auto t : range(axes.begin() + 1, axes.end())) - { - std::transform(cur_pair[1].begin(), - cur_pair[1].end(), - cur_pair[1].begin(), - [t](auto r) { return r > t ? r - 1 : r; }); - } - - return op; - } - - instruction_ref process_pair(const onnx_parser::node_info& info, - instruction_ref op1, - instruction_ref op2, - const int_mat& map_mat, - size_t input_idx, - int_mat& cur_pair) const - { - // Label is present in current two terms and somewhere in subsequent terms - std::vector batch_axes; - // Label is present in only left term - std::vector left_only; - // Label is present in only right term - std::vector right_only; - // Label is present in current two terms, but not in the subsequent terms - std::vector sum_axes; - - auto not_neg_one = [](auto i) { return i != -1; }; - // Categorize axes according to label distribution in equation - for(int d = 0; d < map_mat[0].size(); ++d) - { - // The label is present in both terms of cur_pair - if(all_of(extract_column(cur_pair, d, 0, cur_pair.size()), not_neg_one)) - { - // The label is present in at least one of the subsequent terms - if(any_of(extract_column(map_mat, d, input_idx + 1, map_mat.size()), not_neg_one)) - batch_axes.push_back(d); - else - sum_axes.push_back(d); - } - // The label is missing in one or both of the cur_pair - else - { - if(cur_pair[0][d] >= 0) - left_only.push_back(d); - else if(cur_pair[1][d] >= 0) - right_only.push_back(d); - else - batch_axes.push_back(d); - } - } - - // Permute the inputs so batch_axes are outermost axes and sum_axes are innermost axes - auto&& perm = concat_vectors(batch_axes, left_only, right_only, sum_axes); - std::vector perm64(perm.begin(), perm.end()); - op1 = apply_transpose_op(info, op1, perm64, cur_pair[0]); - op2 = apply_transpose_op(info, op2, perm64, cur_pair[1]); - - auto new_batch_axes = arange(0, batch_axes.size()); - auto new_sum_axes = arange(perm.size() - sum_axes.size(), perm.size()); - - auto common_labels = set_union(new_batch_axes, new_sum_axes); - std::tie(op1, op2) = apply_broadcast_op(info, op1, op2, common_labels); - - auto op = batch_dot(info, cur_pair, op1, op2, new_batch_axes, new_sum_axes); - - return apply_transpose_op(info, op, invert_permutation(perm64), cur_pair[1]); - } - - instruction_ref batch_dot(const onnx_parser::node_info& info, - int_mat& cur_pair, - instruction_ref op1, - instruction_ref op2, - const std::vector& batch_axes, - const std::vector& sum_axes) const - { - auto op1_lens = op1->get_shape().lens(); - auto op2_lens = op2->get_shape().lens(); - - std::vector dims1{static_cast(calc_dim(batch_axes, op1_lens)), - -1, - static_cast(calc_dim(sum_axes, op1_lens))}; - std::vector dims2{static_cast(calc_dim(batch_axes, op2_lens)), - -1, - static_cast(calc_dim(sum_axes, op2_lens))}; - - op1 = info.add_instruction(make_op("reshape", {{"dims", dims1}}), op1); - op2 = info.add_instruction(make_op("reshape", {{"dims", dims2}}), op2); - op2 = info.add_instruction(make_op("transpose", {{"permutation", {0, 2, 1}}}), op2); - instruction_ref op = info.add_instruction(make_op("dot"), op1, op2); - - std::vector new_lens(op1_lens.size(), 1); - std::transform(op1_lens.begin(), - op1_lens.begin() + (new_lens.size() - sum_axes.size()), - op2_lens.begin(), - new_lens.begin(), - [](auto len1, auto len2) { return std::max(len1, len2); }); - - op = info.add_instruction(make_op("reshape", {{"dims", new_lens}}), op); - - // compute output row - std::transform(cur_pair[0].begin(), - cur_pair[0].end(), - cur_pair[1].begin(), - cur_pair[1].begin(), - [](int lhs, int rhs) { return std::max(lhs, rhs); }); - for(int a : sum_axes) - cur_pair[1][a] = -1; - - return op; - } - - instruction_ref finalize_output(const onnx_parser::node_info& info, - instruction_ref op, - const int_mat& map_mat, - int_mat& cur_pair) const - { - if(any_of(map_mat.back(), [](auto i) { return i >= 0; })) - { - cur_pair[1] = map_mat.back(); - std::vector red; - for(int d = 0; d < map_mat[0].size(); ++d) - { - if(cur_pair[0][d] > 0 and cur_pair[1][d] == -1) - red.push_back(d); - } - - op = apply_reduce_sum_op(info, op, red, cur_pair[1]); - } - - return squeeze_transpose(info, cur_pair, op, map_mat.back()); - } - - // Permutes the labels so they are in alphabetical order and expands the input dimensions to - // match the number of unique labels in the entire equation. - instruction_ref transpose_unsqueeze(const onnx_parser::node_info& info, - int_mat& cur_pair, - instruction_ref op) const - { - std::vector perm; - std::vector unsq_axes; - - for(auto i = 0; i < cur_pair[1].size(); ++i) - { - if(cur_pair[1][i] == -1) - // unsqueeze the dimensions corresponding to the missing labels - unsq_axes.push_back(i); - else - // permute the rest - perm.push_back(cur_pair[1][i]); - } - - std::vector perm64(perm.begin(), perm.end()); - op = apply_transpose_op(info, op, perm64, perm); - - // compute output row - for(auto axis : unsq_axes) - { - perm.insert(perm.begin() + axis, -1); - } - cur_pair[1] = perm; - - return info.add_instruction(make_op("unsqueeze", {{"axes", unsq_axes}}), op); - } - - // Reverts the effects of transpose_unsqueeze (adjusts the output so it fits the equation) - instruction_ref squeeze_transpose(const onnx_parser::node_info& info, - int_mat& cur_pair, - instruction_ref op, - std::vector row_output) const - { - std::vector sq_axes; - std::vector perm; - - for(auto i = 0; i < row_output.size(); ++i) - { - if(row_output[i] == -1) - // squeeze the dimensions corresponding to the missing labels - sq_axes.push_back(i); - else - // permute the rest - perm.push_back(row_output[i]); - } - - op = info.add_instruction(make_op("squeeze", {{"axes", sq_axes}}), op); - - if(not perm.empty()) - { - std::vector perm64(perm.begin(), perm.end()); - op = apply_transpose_op(info, op, invert_permutation(perm64), perm); - // compute output row - for(auto axis : sq_axes) - { - perm.insert(perm.begin() + axis, -1); - } - cur_pair[1] = perm; - } - - return op; - } - - instruction_ref apply_transpose_op(const onnx_parser::node_info& info, - instruction_ref op, - const std::vector& perm, - std::vector& row) const - { - op = info.add_instruction(make_op("transpose", {{"permutation", perm}}), op); - // compute output row - row = reorder_dims(row, perm); - - return op; - } - - std::pair - apply_broadcast_op(const onnx_parser::node_info& info, - instruction_ref opl, - instruction_ref opr, - const std::vector& common_labels) const - { - std::pair ret; - - auto llens = opl->get_shape().lens(); - auto rlens = opr->get_shape().lens(); - - bool lbc = false; - bool rbc = false; - for(auto l : common_labels) - { - if(llens[l] == 1 and rlens[l] == 1) - continue; - - if(llens[l] == 1) - { - lbc = true; - llens[l] = rlens[l]; - } - - if(rlens[l] == 1) - { - rbc = true; - rlens[l] = llens[l]; - } - } - - if(lbc) - opl = info.add_instruction(make_op("multibroadcast", {{"out_lens", llens}}), opl); - if(rbc) - opr = info.add_instruction(make_op("multibroadcast", {{"out_lens", rlens}}), opr); - - ret.first = opl; - ret.second = opr; - return ret; - } - - instruction_ref apply_reduce_sum_op(const onnx_parser::node_info& info, - instruction_ref op, - const std::vector& axes, - std::vector& row) const - { - if(axes.empty()) - return op; - - for(int a : axes) - row[a] = -1; - - return info.add_instruction(make_op("reduce_sum", {{"axes", axes}}), op); - } - - // Utility - - int_mat make_matrix(int cur_pair, int cols, int fill_value) const - { - return {static_cast(cur_pair), std::vector(cols, fill_value)}; - } - - std::vector extract_column(int_mat map_mat, int col_idx, int row_begin, int row_end) const - { - std::vector ret; - ret.reserve(row_end - row_begin); - - std::transform(map_mat.begin() + row_begin, - map_mat.begin() + row_end, - std::back_inserter(ret), - [col_idx](const auto& x) { return x[col_idx]; }); - - return ret; - } - - std::vector set_union(const std::vector& lhs, const std::vector& rhs) const - { - std::vector ret; - std::set_union(lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), std::back_inserter(ret)); - - return ret; - } - - std::vector set_difference(const std::vector& lhs, const std::vector& rhs) const - { - std::vector ret; - std::set_difference( - lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), std::back_inserter(ret)); - - return ret; - } - - // Equivalent to numpy.arange without the step parameter - std::vector arange(int start_value, int end_value) const - { - std::vector ret(end_value - start_value); - std::iota(ret.begin(), ret.end(), start_value); - return ret; - } - - template - Vec concat_vectors(Vec vec, Vecs&&... vecs) const - { - size_t reserve_size = vec.size(); - each_args([&](auto&& v) { reserve_size += v.size(); }, vecs...); - - vec.reserve(reserve_size); - each_args([&](auto&& v) { vec.insert(vec.end(), v.begin(), v.end()); }, vecs...); - - return vec; - } - - size_t calc_dim(const std::vector& axes, const std::vector& lens) const - { - return std::accumulate( - axes.begin(), axes.end(), 1, [&](auto acc, auto axis) { return acc * lens[axis]; }); - }; -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_expand.cpp b/docker/rocm/migraphx/onnx/parse_expand.cpp deleted file mode 100644 index e468cad10..000000000 --- a/docker/rocm/migraphx/onnx/parse_expand.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_expand : op_parser -{ - std::vector operators() const { return {{"Expand"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - migraphx::argument arg_s = args[1]->eval(); - if(arg_s.empty()) - { - // variable dims input - return info.add_instruction(make_op("broadcast_with_dims"), args[0], args[1]); - } - else - { - const shape& shape_0 = args[0]->get_shape(); - if(shape_0.dynamic()) - { - MIGRAPHX_THROW( - "PARSE_EXPAND: dynamic input tensor with fixed dims input not supported"); - } - const auto& in_lens = shape_0.lens(); - std::vector dims; - arg_s.visit([&](auto input) { dims.assign(input.begin(), input.end()); }); - auto out_lens = compute_broadcasted_lens(in_lens, dims); - return info.add_instruction(make_op("multibroadcast", {{"out_lens", out_lens}}), - args[0]); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_eyelike.cpp b/docker/rocm/migraphx/onnx/parse_eyelike.cpp deleted file mode 100644 index c29e4dcb9..000000000 --- a/docker/rocm/migraphx/onnx/parse_eyelike.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_eyelike : op_parser -{ - std::vector operators() const { return {{"EyeLike"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser&, - const onnx_parser::node_info& info, - std::vector args) const - { - auto input_shape = args[0]->get_shape(); - auto input_lens = input_shape.lens(); - if(input_lens.size() != 2) - { - MIGRAPHX_THROW("EYELIKE: tensor input not of rank 2"); - } - std::ptrdiff_t num_rows = input_lens.front(); - std::ptrdiff_t num_cols = input_lens.back(); - - shape::type_t output_type = args[0]->get_shape().type(); - if(contains(info.attributes, "dtype")) - { - output_type = get_type(info.attributes.at("dtype").i()); - } - - std::ptrdiff_t k = 0; - if(contains(info.attributes, "k")) - { - k = info.attributes.at("k").i(); - } - if(k >= 0) - { - if(k >= num_cols) - { - std::ostringstream oss; - oss << "EYELIKE: positive k out of bounds, k = " << k << " num_cols = " << num_cols; - MIGRAPHX_THROW(oss.str()); - } - } - else - { - if(std::abs(k) >= num_rows) - { - std::ostringstream oss; - oss << "EYELIKE: negative k out of bounds, k = " << k << " num_rows = " << num_cols; - MIGRAPHX_THROW(oss.str()); - } - } - - std::vector eyelike_mat(num_rows * num_cols, 0); - for(std::ptrdiff_t i = 0; i < num_rows; ++i) - { - auto idx = i + k; - if(idx < num_cols and idx >= 0) - eyelike_mat[(num_cols + 1) * i + k] = char{1}; - } - return info.add_literal( - migraphx::literal{migraphx::shape{output_type, input_lens}, eyelike_mat}); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_gather_elements.cpp b/docker/rocm/migraphx/onnx/parse_gather_elements.cpp deleted file mode 100644 index cd67d535f..000000000 --- a/docker/rocm/migraphx/onnx/parse_gather_elements.cpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_gather_elements : op_parser -{ - std::vector operators() const { return {{"GatherElements"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - int axis = 0; - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - - // standardize input data and index - auto arg_data = info.make_contiguous(args[0]); - auto arg_ind = info.make_contiguous(args[1]); - - auto data_s = arg_data->get_shape(); - auto ind_s = arg_ind->get_shape(); - - if(data_s.lens().size() != ind_s.lens().size()) - { - MIGRAPHX_THROW("PARSE_GATHER_ELEMENTS: input data and index must have the same rank!"); - } - - int n_rank = data_s.lens().size(); - int tuned_axis = tune_axis(n_rank, axis, opd.onnx_name); - - auto axis_stride = data_s.strides()[tuned_axis]; - int64_t data_elem_num = data_s.elements(); - // reshape the input data as one dimension and used as input data - // to the gather operator - arg_data = info.add_instruction(make_op("reshape", {{"dims", {data_elem_num}}}), arg_data); - - std::size_t elem_num = ind_s.elements(); - std::vector ind_index(elem_num); - std::iota(ind_index.begin(), ind_index.end(), 0); - - // convert index in input indices to that in input data - std::vector data_indices(elem_num); - std::transform(ind_index.begin(), ind_index.end(), data_indices.begin(), [&](auto i) { - return data_s.index(ind_s.multi(i)); - }); - - std::vector vec_axis_ind(elem_num); - std::transform(ind_index.begin(), ind_index.end(), vec_axis_ind.begin(), [&](auto i) { - return ind_s.multi(i)[tuned_axis]; - }); - - auto l_shape_idx = - info.add_literal(literal(ind_s, data_indices.begin(), data_indices.end())); - auto l_dim_idx = info.add_literal(literal(ind_s, vec_axis_ind.begin(), vec_axis_ind.end())); - auto l_stride = info.add_literal(literal{{ind_s.type(), {1}}, {axis_stride}}); - l_stride = - info.add_instruction(make_op("multibroadcast", {{"out_lens", ind_s.lens()}}), l_stride); - auto dim_diff = info.add_instruction(make_op("sub"), arg_ind, l_dim_idx); - auto delta = info.add_instruction(make_op("mul"), dim_diff, l_stride); - auto ind = info.add_instruction(make_op("add"), l_shape_idx, delta); - - auto op = make_op("gather", {{"axis", 0}}); - return info.add_instruction(op, arg_data, ind); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_gelu.cpp b/docker/rocm/migraphx/onnx/parse_gelu.cpp deleted file mode 100644 index af9a5635f..000000000 --- a/docker/rocm/migraphx/onnx/parse_gelu.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -instruction_ref parse_gelu_erf(const onnx_parser::node_info& info, instruction_ref x) -{ - auto x_type = x->get_shape().type(); - auto half = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {0.5f}}); - auto one = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {1.0f}}); - auto sqrt2 = - info.add_literal(migraphx::literal{migraphx::shape{x_type}, {static_cast(M_SQRT2)}}); - auto mul_half = info.add_common_op("mul", x, half); - auto div = info.add_common_op("div", x, sqrt2); - auto erf = info.add_instruction(migraphx::make_op("erf"), div); - auto add_one = info.add_common_op("add", erf, one); - return info.add_common_op("mul", mul_half, add_one); -} - -instruction_ref parse_gelu_tanh(const onnx_parser::node_info& info, instruction_ref x, bool fast) -{ - auto x_type = x->get_shape().type(); - auto fit_const_val = fast ? 0.035677 : 0.044715; - auto fit_const = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {fit_const_val}}); - auto sqrt_2_rpi_val = fast ? 0.797885 : sqrt(M_2_PI); - auto sqrt_2_rpi = - info.add_literal(migraphx::literal{migraphx::shape{x_type}, {sqrt_2_rpi_val}}); - auto one = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {1.0f}}); - auto half = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {0.5f}}); - auto three = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {3.0f}}); - - // [0.044715|0.035677] * x^3 - auto pow0 = info.add_common_op("pow", x, three); - auto mul0 = info.add_common_op("mul", pow0, fit_const); - instruction_ref tanh_in; - if(fast) - { - // approx = 0.797885 * x + 0.035677 * x^3 - auto mul1 = info.add_common_op("mul", sqrt_2_rpi, x); - tanh_in = info.add_common_op("add", mul0, mul1); - } - else - { - // approx = sqrt(2/pi) * (x + 0.044715 * x^3 - auto add0 = info.add_common_op("add", mul0, x); - tanh_in = info.add_common_op("mul", add0, sqrt_2_rpi); - } - - // 0.5 * x * (1 + Tanh(approx)) - auto tanh0 = info.add_instruction(migraphx::make_op("tanh"), tanh_in); - auto add1 = info.add_common_op("add", tanh0, one); - auto mul2 = info.add_common_op("mul", x, half); - return info.add_common_op("mul", add1, mul2); -} - -struct parse_gelu : op_parser -{ - std::vector operators() const { return {{"BiasGelu"}, {"FastGelu"}, {"Gelu"}}; } - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - std::string approximate = "none"; - auto x = args[0]; - auto x_type = x->get_shape().type(); - auto fast = false; - if(not is_type_float(x_type)) - { - MIGRAPHX_THROW("PARSE_GELU: input tensor is not a floating type"); - } - - if(contains(info.attributes, "approximate")) - { - approximate = info.attributes.at("approximate").s(); - } - - if(opd.onnx_name == "FastGelu") - { - if(x_type == migraphx::shape::double_type) - { - MIGRAPHX_THROW("PARSE_GELU: FastGelu can't accept input with double precision"); - } - - // FastGelu uses tanh approximation - approximate = "tanh"; - fast = true; - } - - if(args.size() > 1 and args.at(1)->name() != "undefined") - { - auto y = args[1]; - auto y_type = y->get_shape().type(); - if(y_type != x_type) - { - MIGRAPHX_THROW("PARSE_GELU: mismatching input tensor types"); - } - x = info.add_common_op("add", x, y); - } - - if(approximate == "tanh") - { - return parse_gelu_tanh(info, x, fast); - } - else - { - return parse_gelu_erf(info, x); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_gemm.cpp b/docker/rocm/migraphx/onnx/parse_gemm.cpp deleted file mode 100644 index b67c3304e..000000000 --- a/docker/rocm/migraphx/onnx/parse_gemm.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_gemm : op_parser -{ - std::vector operators() const { return {{"Gemm"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - auto a_arg = args[0]; - auto b_arg = args[1]; - if(a_arg->get_shape().ndim() != 2 or b_arg->get_shape().ndim() != 2) - { - MIGRAPHX_THROW("PARSE_GEMM: A and B should be rank 2, A is rank " + - std::to_string(a_arg->get_shape().ndim()) + ", B is rank " + - std::to_string(b_arg->get_shape().ndim())); - } - - float alpha = 1.0f; - float beta = 1.0f; - bool trans_a = false; - bool trans_b = false; - if(contains(info.attributes, "alpha")) - { - alpha = parser.parse_value(info.attributes.at("alpha")).at(); - } - if(contains(info.attributes, "beta")) - { - beta = parser.parse_value(info.attributes.at("beta")).at(); - } - if(contains(info.attributes, "transA")) - { - trans_a = parser.parse_value(info.attributes.at("transA")).at(); - } - if(contains(info.attributes, "transB")) - { - trans_b = parser.parse_value(info.attributes.at("transB")).at(); - } - - std::vector perm = {1, 0}; - auto dot_type = a_arg->get_shape().type(); - if(alpha != 1.0f) - { - auto alpha_literal = info.add_literal(alpha); - a_arg = info.add_broadcastable_binary_op("mul", alpha_literal, a_arg); - - if(a_arg->get_shape().type() != dot_type) - { - a_arg = - info.add_instruction(make_op("convert", {{"target_type", dot_type}}), a_arg); - } - } - - a_arg = (trans_a) - ? info.add_instruction(make_op("transpose", {{"permutation", perm}}), a_arg) - : a_arg; - b_arg = (trans_b) - ? info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[1]) - : args[1]; - - auto dot_ins = info.add_instruction(make_op("dot"), a_arg, b_arg); - - if(args.size() == 3) - { - if(not float_equal(beta, 0.0f)) - { - auto c_arg = args[2]; - if(dot_ins->get_shape().dynamic()) - { - c_arg = info.add_instruction(make_op("multibroadcast"), args[2], dot_ins); - } - else - { - auto out_lens = a_arg->get_shape().lens(); - out_lens.back() = b_arg->get_shape().lens().back(); - auto c_lens = c_arg->get_shape().lens(); - if(not std::equal( - out_lens.begin(), out_lens.end(), c_lens.begin(), c_lens.end())) - { - c_arg = info.add_instruction( - make_op("multibroadcast", {{"out_lens", out_lens}}), args[2]); - } - } - - if(not float_equal(beta, 1.0f)) - { - auto beta_literal = info.add_literal(beta); - c_arg = info.add_broadcastable_binary_op("mul", c_arg, beta_literal); - if(c_arg->get_shape().type() != dot_type) - { - c_arg = info.add_instruction( - make_op("convert", {{"target_type", dot_type}}), c_arg); - } - } - - return info.add_instruction(make_op("add"), dot_ins, c_arg); - } - } - return dot_ins; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_generic_op.cpp b/docker/rocm/migraphx/onnx/parse_generic_op.cpp deleted file mode 100644 index 023a3a41f..000000000 --- a/docker/rocm/migraphx/onnx/parse_generic_op.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_generic_op : op_parser -{ - std::vector operators() const - { - // clang-format off - return {{"Abs", "abs"}, - {"Acos", "acos"}, - {"Acosh", "acosh"}, - {"Asin", "asin"}, - {"Asinh", "asinh"}, - {"Atan", "atan"}, - {"Atanh", "atanh"}, - {"Ceil", "ceil"}, - {"Concat", "concat"}, - {"Cos", "cos"}, - {"Cosh", "cosh"}, - {"Elu", "elu"}, - {"Erf", "erf"}, - {"Exp", "exp"}, - {"Flatten", "flatten"}, - {"Floor", "floor"}, - {"Gather", "gather"}, - {"GatherND", "gathernd"}, - {"Identity", "identity"}, - {"IsNaN", "isnan"}, - {"LeakyRelu", "leaky_relu"}, - {"Log", "log"}, - {"LRN", "lrn"}, - {"Neg", "neg"}, - {"Reciprocal", "recip"}, - {"Relu", "relu"}, - {"Round", "nearbyint"}, - {"Sigmoid", "sigmoid"}, - {"Sign", "sign"}, - {"Sin", "sin"}, - {"Sinh", "sinh"}, - {"Sqrt", "sqrt"}, - {"Tan", "tan"}, - {"Tanh", "tanh"}, - {"Not", "not"}}; - // clang-format on - } - - bool needs_contiguous(const std::string& op_name) const - { - return contains({"flatten", "gather", "scatter"}, op_name); - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - auto op = parser.load(opd.op_name, info); - if(needs_contiguous(opd.op_name)) - { - std::transform(args.begin(), args.end(), args.begin(), [&](auto arg) { - return info.make_contiguous(arg); - }); - } - return info.add_instruction(op, args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_greaterorequal.cpp b/docker/rocm/migraphx/onnx/parse_greaterorequal.cpp deleted file mode 100644 index 469af9b04..000000000 --- a/docker/rocm/migraphx/onnx/parse_greaterorequal.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_greaterorequal : op_parser -{ - std::vector operators() const { return {{"GreaterOrEqual"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto in_res = info.add_broadcastable_binary_op("less", args[0], args[1]); - if(in_res->get_shape().type() != shape::bool_type) - { - in_res = info.add_instruction(make_op("convert", {{"target_type", shape::bool_type}}), - in_res); - } - return info.add_instruction(make_op("not"), in_res); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_gridsample.cpp b/docker/rocm/migraphx/onnx/parse_gridsample.cpp deleted file mode 100644 index 63a6833bf..000000000 --- a/docker/rocm/migraphx/onnx/parse_gridsample.cpp +++ /dev/null @@ -1,717 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct grid_sampler -{ - std::string m_padding; - bool m_align_corners; - - instruction_ref m_input; - instruction_ref m_grid; - - size_t m_batch{1}; - size_t m_channel{1}; - size_t m_in_height{1}; - size_t m_in_width{1}; - size_t m_out_height{1}; - size_t m_out_width{1}; - migraphx::shape m_nc_shape; - - instruction_ref m_one_l; - instruction_ref m_two_l; - instruction_ref m_zero_l; - instruction_ref m_minus_half_l; - instruction_ref m_width_l; - instruction_ref m_width_max_l; - instruction_ref m_height_l; - instruction_ref m_height_max_l; - instruction_ref m_unnorm_x; - instruction_ref m_unnorm_y; - - grid_sampler(const instruction_ref& input, - const instruction_ref& grid, - bool align, - std::string&& padding, - const onnx_parser::node_info& info) - : m_padding(std::move(padding)), m_align_corners(align), m_input(input), m_grid(grid) - { - auto i_lens = input->get_shape().lens(); - m_batch = i_lens.at(0); - m_channel = i_lens.at(1); - m_in_height = i_lens.at(2); - m_in_width = i_lens.at(3); - auto g_lens = grid->get_shape().lens(); - m_out_height = g_lens.at(1); - m_out_width = g_lens.at(2); - auto type = m_grid->get_shape().type(); - m_nc_shape = migraphx::shape{type, {1, 2}}; - m_zero_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {0.0f}}); - m_one_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {1.0f}}); - m_two_l = - info.add_literal(migraphx::literal{migraphx::shape{migraphx::shape::int64_type}, {2}}); - m_minus_half_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {-0.5f}}); - m_width_max_l = - info.add_literal(migraphx::literal{migraphx::shape{type}, {m_in_width - 1}}); - m_width_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {m_in_width}}); - m_height_max_l = - info.add_literal(migraphx::literal{migraphx::shape{type}, {m_in_height - 1}}); - m_height_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {m_in_height}}); - - auto x_coords = info.add_instruction( - make_op("slice", {{"axes", {3}}, {"starts", {0}}, {"ends", {1}}}), m_grid); - - auto y_coords = info.add_instruction( - make_op("slice", {{"axes", {3}}, {"starts", {1}}, {"ends", {2}}}), m_grid); - - x_coords = info.add_instruction(make_op("squeeze", {{"axes", {3}}}), x_coords); - y_coords = info.add_instruction(make_op("squeeze", {{"axes", {3}}}), y_coords); - m_unnorm_x = unnormalize(info, x_coords, m_in_width); - m_unnorm_y = unnormalize(info, y_coords, m_in_height); - - if(m_padding == "reflection") - { - auto corner_start = m_align_corners ? m_zero_l : m_minus_half_l; - m_unnorm_x = reflect_coordinates( - info, m_unnorm_x, m_align_corners ? m_width_max_l : m_width_l, corner_start); - m_unnorm_y = reflect_coordinates( - info, m_unnorm_y, m_align_corners ? m_height_max_l : m_height_l, corner_start); - m_unnorm_x = info.add_common_op("clip", m_unnorm_x, m_zero_l, m_width_max_l); - m_unnorm_y = info.add_common_op("clip", m_unnorm_y, m_zero_l, m_height_max_l); - } - - if(m_padding == "border") - { - m_unnorm_x = info.add_common_op("clip", m_unnorm_x, m_zero_l, m_width_max_l); - m_unnorm_y = info.add_common_op("clip", m_unnorm_y, m_zero_l, m_height_max_l); - } - } - - instruction_ref reflect_coordinates(const onnx_parser::node_info& info, - instruction_ref coords, - instruction_ref size, - instruction_ref corner_start) const - { - auto index_align_corner = info.add_common_op("sub", corner_start, coords); - index_align_corner = info.add_common_op("abs", index_align_corner); - auto size_times = info.add_common_op("floor", index_align_corner); - size_times = info.add_common_op("div", size_times, size); - size_times = info.add_common_op("floor", size_times); - auto cond = info.add_common_op("mod", size_times, m_two_l); - cond = info.add_common_op("equal", cond, m_zero_l); - auto extra = info.add_common_op("mul", size_times, size); - extra = info.add_common_op("sub", index_align_corner, extra); - auto cond_true = info.add_common_op("add", extra, corner_start); - auto cond_false = info.add_common_op("sub", size, extra); - cond_false = info.add_common_op("add", cond_false, corner_start); - return info.add_common_op("where", cond, cond_true, cond_false); - } - - instruction_ref unnormalize(const onnx_parser::node_info& info, - const instruction_ref& coords_t, - float size) const - { - auto unnorm = info.add_common_op("add", coords_t, m_one_l); - if(m_align_corners) - { - // unnorm_x = (x + 1) * (size - 1) / 2 - auto mul_const = info.add_literal( - migraphx::literal{migraphx::shape{coords_t->get_shape().type()}, {(size - 1) / 2}}); - unnorm = info.add_common_op("mul", unnorm, mul_const); - } - else - { - // unnorm_x = -0.5 + (x + 1) * size / 2 - auto mul_const = info.add_literal( - migraphx::literal{migraphx::shape{coords_t->get_shape().type()}, {size / 2}}); - unnorm = info.add_common_op("mul", unnorm, mul_const); - unnorm = info.add_common_op("add", unnorm, m_minus_half_l); - } - return unnorm; - } - - static instruction_ref concat_on_first_dim(const onnx_parser::node_info& info, - std::vector instructions) - { - return std::accumulate( - std::next(instructions.begin()), - instructions.end(), - instructions.front(), - [&info](auto& ret, auto& ins) { - return info.add_instruction(make_op("concat", {{"axis", 0}}), ret, ins); - }); - } - - static instruction_ref concat_on_dim(const onnx_parser::node_info& info, - std::array instructions, - int64_t dim) - { - return std::accumulate( - std::next(instructions.begin()), - instructions.end(), - instructions.front(), - [&info, &dim](auto& ret, auto& ins) { - return info.add_instruction(make_op("concat", {{"axis", dim}}), ret, ins); - }); - } - - inline bool has_border_padding() const { return m_padding == "border"; } -}; - -struct nearest_sampler : grid_sampler -{ - instruction_ref m_round_x; - instruction_ref m_round_y; - - nearest_sampler(const instruction_ref& input, - const instruction_ref& grid, - bool align, - std::string&& padding, - const onnx_parser::node_info& info) - : grid_sampler(input, grid, align, std::move(padding), info), - m_round_x(info.add_common_op("nearbyint", m_unnorm_x)), - m_round_y(info.add_common_op("nearbyint", m_unnorm_y)) - { - } - - instruction_ref sample(const onnx_parser::node_info& info) - { - std::vector hw_indices; - std::vector nc_values; - const static auto nhw_shape = migraphx::shape{migraphx::shape::int64_type, {1, 3}}; - bool validate = not has_border_padding(); - dfor(m_batch, m_out_height, m_out_width)([&](auto n, auto h, auto w) { - auto nhw = info.add_literal(migraphx::literal{nhw_shape, {n, h, w}}); - for(size_t c = 0; c < m_channel; c++) - { - hw_indices.push_back(nhw); - nc_values.push_back(info.add_literal(migraphx::literal{m_nc_shape, {n, c}})); - } - }); - - auto hw_indices_t = concat_on_first_dim(info, hw_indices); - auto h_samples = info.add_instruction(make_op("gathernd"), m_round_y, hw_indices_t); - auto w_samples = info.add_instruction(make_op("gathernd"), m_round_x, hw_indices_t); - - instruction_ref validation; - if(validate) - { - auto h_clip = info.add_common_op("clip", h_samples, m_zero_l, m_height_max_l); - auto w_clip = info.add_common_op("clip", w_samples, m_zero_l, m_width_max_l); - auto h_valid = info.add_common_op("equal", h_samples, h_clip); - auto w_valid = info.add_common_op("equal", w_samples, w_clip); - validation = info.add_common_op("logical_and", h_valid, w_valid); - h_samples = h_clip; - w_samples = w_clip; - } - - auto nc = concat_on_first_dim(info, nc_values); - h_samples = info.add_instruction( - make_op("reshape", {{"dims", {h_samples->get_shape().elements(), 1}}}), h_samples); - - w_samples = info.add_instruction( - make_op("reshape", {{"dims", {w_samples->get_shape().elements(), 1}}}), w_samples); - - auto indices_t = - info.add_instruction(make_op("concat", {{"axis", 1}}), h_samples, w_samples); - indices_t = info.add_instruction(make_op("concat", {{"axis", 1}}), nc, indices_t); - - auto samples = info.add_instruction(make_op("gathernd"), m_input, indices_t); - if(validate) - { - samples = info.add_common_op("where", validation, samples, m_zero_l); - } - - samples = info.add_instruction( - make_op("reshape", {{"dims", {m_batch, m_out_height, m_out_width, m_channel}}}), - samples); - samples = - info.add_instruction(make_op("transpose", {{"permutation", {0, 3, 1, 2}}}), samples); - samples = info.add_instruction( - make_op("convert", {{"target_type", m_input->get_shape().type()}}), samples); - return samples; - } -}; - -struct linear_sampler : grid_sampler -{ - instruction_ref m_floor_x; - instruction_ref m_floor_y; - instruction_ref m_ceil_x; - instruction_ref m_ceil_y; - std::array m_corner_weights; - - linear_sampler(const instruction_ref& input, - const instruction_ref& grid, - bool align, - std::string&& padding, - const onnx_parser::node_info& info) - : grid_sampler(input, grid, align, std::move(padding), info), - m_floor_x(info.add_common_op("floor", m_unnorm_x)), - m_floor_y(info.add_common_op("floor", m_unnorm_y)), - m_ceil_x(info.add_common_op("add", m_floor_x, m_one_l)), - m_ceil_y(info.add_common_op("add", m_floor_y, m_one_l)) - { - auto fract_x = info.add_common_op("sub", m_unnorm_x, m_floor_x); - auto fract_y = info.add_common_op("sub", m_unnorm_y, m_floor_y); - auto one_minus_fract_x = info.add_common_op("sub", m_one_l, fract_x); - auto one_minus_fract_y = info.add_common_op("sub", m_one_l, fract_y); - m_corner_weights[0] = info.add_common_op("mul", one_minus_fract_y, one_minus_fract_x); - m_corner_weights[1] = info.add_common_op("mul", one_minus_fract_y, fract_x); - m_corner_weights[2] = info.add_common_op("mul", fract_y, one_minus_fract_x); - m_corner_weights[3] = info.add_common_op("mul", fract_y, fract_x); - } - - instruction_ref sample(const onnx_parser::node_info& info) - { - std::vector weight_indices; - std::vector xy_indices; - std::vector nc_values; - - const static auto nhw_shape = migraphx::shape{migraphx::shape::int64_type, {1, 3}}; - dfor(m_batch, m_out_height, m_out_width)([&](auto n, auto h, auto w) { - auto nhw = info.add_literal(migraphx::literal{nhw_shape, {n, h, w}}); - weight_indices.push_back(nhw); - for(size_t c = 0; c < m_channel; c++) - { - xy_indices.push_back(nhw); - nc_values.push_back(info.add_literal(migraphx::literal{m_nc_shape, {n, c}})); - } - }); - - auto xy_indices_t = concat_on_first_dim(info, xy_indices); - auto y0_samples = info.add_instruction(make_op("gathernd"), m_floor_y, xy_indices_t); - auto x0_samples = info.add_instruction(make_op("gathernd"), m_floor_x, xy_indices_t); - auto y1_samples = info.add_instruction(make_op("gathernd"), m_ceil_y, xy_indices_t); - auto x1_samples = info.add_instruction(make_op("gathernd"), m_ceil_x, xy_indices_t); - - auto validate_samples = [&](auto& samples, auto& max) { - auto clip = info.add_common_op("clip", samples, m_zero_l, max); - auto validation = info.add_common_op("equal", samples, clip); - samples = clip; - return validation; - }; - - auto y0_validation = validate_samples(y0_samples, m_height_max_l); - auto x0_validation = validate_samples(x0_samples, m_width_max_l); - auto y1_validation = validate_samples(y1_samples, m_height_max_l); - auto x1_validation = validate_samples(x1_samples, m_width_max_l); - - y0_samples = info.add_instruction( - make_op("reshape", {{"dims", {y0_samples->get_shape().elements(), 1}}}), y0_samples); - x0_samples = info.add_instruction( - make_op("reshape", {{"dims", {x0_samples->get_shape().elements(), 1}}}), x0_samples); - y1_samples = info.add_instruction( - make_op("reshape", {{"dims", {y1_samples->get_shape().elements(), 1}}}), y1_samples); - x1_samples = info.add_instruction( - make_op("reshape", {{"dims", {x1_samples->get_shape().elements(), 1}}}), x1_samples); - - auto nc = concat_on_first_dim(info, nc_values); - - auto make_corner_indices = [&](auto& x, auto& y) { - auto hw = info.add_instruction(make_op("concat", {{"axis", 1}}), y, x); - return info.add_instruction(make_op("concat", {{"axis", 1}}), nc, hw); - }; - std::array corner_indices{make_corner_indices(x0_samples, y0_samples), - make_corner_indices(x1_samples, y0_samples), - make_corner_indices(x0_samples, y1_samples), - make_corner_indices(x1_samples, y1_samples)}; - - std::array corner_validations{ - info.add_common_op("logical_and", x0_validation, y0_validation), - info.add_common_op("logical_and", x1_validation, y0_validation), - info.add_common_op("logical_and", x0_validation, y1_validation), - info.add_common_op("logical_and", x1_validation, y1_validation)}; - - std::array corner_samples; - auto weight_index_t = concat_on_first_dim(info, weight_indices); - weight_index_t = info.add_instruction( - make_op("reshape", {{"dims", {weight_indices.size(), 3}}}), weight_index_t); - - std::transform(corner_indices.begin(), - corner_indices.end(), - corner_validations.begin(), - corner_samples.begin(), - [&](const auto& indices, const auto& validations) { - auto samples = - info.add_instruction(make_op("gathernd"), m_input, indices); - return info.add_common_op("where", validations, samples, m_zero_l); - }); - - std::transform(corner_samples.begin(), - corner_samples.end(), - m_corner_weights.begin(), - corner_samples.begin(), - [&](const auto& samples, const auto& weights) { - auto weights_t = - info.add_instruction(make_op("gathernd"), weights, weight_index_t); - return info.add_instruction(make_op("mul"), samples, weights_t); - }); - - auto samples = std::accumulate( - std::next(corner_samples.begin()), - corner_samples.end(), - corner_samples.front(), - [&](auto acc, auto s) { return info.add_instruction(make_op("add"), acc, s); }); - - samples = info.add_instruction( - make_op("reshape", {{"dims", {m_batch, m_out_height, m_out_width, m_channel}}}), - samples); - samples = - info.add_instruction(make_op("transpose", {{"permutation", {0, 3, 1, 2}}}), samples); - samples = info.add_instruction( - make_op("convert", {{"target_type", m_input->get_shape().type()}}), samples); - return samples; - } -}; - -struct bicubic_sampler : grid_sampler -{ - instruction_ref m_a_l; - instruction_ref m_aplus2_l; - instruction_ref m_aplus3_l; - instruction_ref m_4a_l; - instruction_ref m_5a_l; - instruction_ref m_8a_l; - std::array m_x_weights; - std::array m_y_weights; - std::array m_x_corners; - std::array m_y_corners; - - bicubic_sampler(const instruction_ref& input, - const instruction_ref& grid, - bool align, - std::string&& padding, - const onnx_parser::node_info& info) - : grid_sampler(input, grid, align, std::move(padding), info) - { - auto type = m_grid->get_shape().type(); - m_a_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {-0.75}}); - m_aplus2_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {1.25}}); - m_aplus3_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {2.25}}); - m_4a_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {-3.0}}); - m_5a_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {-3.75}}); - m_8a_l = info.add_literal(migraphx::literal{migraphx::shape{type}, {-6.0}}); - auto floor_x = info.add_common_op("floor", m_unnorm_x); - auto floor_y = info.add_common_op("floor", m_unnorm_y); - auto fract_x = info.add_common_op("sub", m_unnorm_x, floor_x); - auto fract_y = info.add_common_op("sub", m_unnorm_y, floor_y); - - m_x_weights[0] = cubic_weight_2(info, info.add_common_op("add", fract_x, m_one_l)); - m_x_weights[1] = cubic_weight_1(info, fract_x); - m_x_weights[2] = cubic_weight_1(info, info.add_common_op("sub", m_one_l, fract_x)); - m_x_weights[3] = cubic_weight_2(info, info.add_common_op("sub", m_two_l, fract_x)); - - m_y_weights[0] = cubic_weight_2(info, info.add_common_op("add", fract_y, m_one_l)); - m_y_weights[1] = cubic_weight_1(info, fract_y); - m_y_weights[2] = cubic_weight_1(info, info.add_common_op("sub", m_one_l, fract_y)); - m_y_weights[3] = cubic_weight_2(info, info.add_common_op("sub", m_two_l, fract_y)); - - m_x_corners[0] = info.add_common_op("sub", floor_x, m_one_l); - m_x_corners[1] = floor_x; - m_x_corners[2] = info.add_common_op("add", floor_x, m_one_l); - m_x_corners[3] = info.add_common_op("add", floor_x, m_two_l); - - m_y_corners[0] = info.add_common_op("sub", floor_y, m_one_l); - m_y_corners[1] = floor_y; - m_y_corners[2] = info.add_common_op("add", floor_y, m_one_l); - m_y_corners[3] = info.add_common_op("add", floor_y, m_two_l); - - if(m_padding == "reflection") - { - auto corner_start = m_align_corners ? m_zero_l : m_minus_half_l; - std::transform( - m_x_corners.begin(), - m_x_corners.end(), - m_x_corners.begin(), - [&](const auto& corner) { - auto tmp = reflect_coordinates( - info, corner, m_align_corners ? m_width_max_l : m_width_l, corner_start); - return info.add_common_op("clip", tmp, m_zero_l, m_width_max_l); - }); - std::transform( - m_y_corners.begin(), - m_y_corners.end(), - m_y_corners.begin(), - [&](const auto& corner) { - auto tmp = reflect_coordinates( - info, corner, m_align_corners ? m_height_max_l : m_height_l, corner_start); - return info.add_common_op("clip", tmp, m_zero_l, m_height_max_l); - }); - } - - if(m_padding == "border") - { - std::transform( - m_x_corners.begin(), m_x_corners.end(), m_x_corners.begin(), [&](auto& corner) { - return info.add_common_op("clip", corner, m_zero_l, m_width_max_l); - }); - std::transform( - m_y_corners.begin(), m_y_corners.end(), m_y_corners.begin(), [&](auto& corner) { - return info.add_common_op("clip", corner, m_zero_l, m_height_max_l); - }); - } - } - - instruction_ref cubic_weight_1(const onnx_parser::node_info& info, - const instruction_ref& ins) const - { - //((A + 2) * fraction - (A + 3)) * fraction * fraction + 1 - auto mul_1 = info.add_common_op("mul", m_aplus2_l, ins); - auto sub = info.add_common_op("sub", mul_1, m_aplus3_l); - auto mul_2 = info.add_common_op("mul", sub, ins); - auto mul_3 = info.add_common_op("mul", mul_2, ins); - return info.add_common_op("add", mul_3, m_one_l); - } - - instruction_ref cubic_weight_2(const onnx_parser::node_info& info, - const instruction_ref& ins) const - { - // ((A * fraction - 5 * A) * fraction + 8 * A) * fraction - (4 * A) - auto mul_1 = info.add_common_op("mul", m_a_l, ins); - auto sub_1 = info.add_common_op("sub", mul_1, m_5a_l); - auto mul_2 = info.add_common_op("mul", sub_1, ins); - auto add = info.add_common_op("add", mul_2, m_8a_l); - auto mul_3 = info.add_common_op("mul", add, ins); - return info.add_common_op("sub", mul_3, m_4a_l); - } - - static instruction_ref compute_weights(const onnx_parser::node_info& info, - const std::vector& weight_indices, - const std::array& weights, - const std::vector& out_lens, - size_t gather_dim) - { - auto weight_indices_t = concat_on_first_dim(info, weight_indices); - weight_indices_t = info.add_instruction( - make_op( - "reshape", - {{"dims", {weight_indices_t->get_shape().elements() / gather_dim, gather_dim}}}), - weight_indices_t); - std::array corner_weights; - std::transform(weights.cbegin(), weights.cend(), corner_weights.begin(), [&](auto& corner) { - auto corner_weight = - info.add_instruction(make_op("gathernd"), corner, weight_indices_t); - return info.add_instruction( - make_op("reshape", {{"dims", {corner_weight->get_shape().elements(), 1}}}), - corner_weight); - }); - auto weights_t = std::accumulate( - std::next(corner_weights.begin()), - corner_weights.end(), - corner_weights.front(), - [&info](auto& acc, auto& ins) { - return info.add_instruction(make_op("concat", {{"axis", 1}}), acc, ins); - }); - return info.add_instruction(make_op("reshape", {{"dims", out_lens}}), weights_t); - } - - instruction_ref sample(const onnx_parser::node_info& info) - { - std::vector x_weight_indices; - std::vector y_weight_indices; - - std::vector inner_x_indices; - std::vector nc_values; - std::vector inner_indices; - const static auto nhw_shape = migraphx::shape{migraphx::shape::int64_type, {3}}; - dfor(m_batch, m_out_height, m_out_width)([&](auto n, auto h, auto w) { - auto nhw = info.add_literal(migraphx::literal{nhw_shape, {n, h, w}}); - x_weight_indices.insert(x_weight_indices.end(), {nhw, nhw, nhw, nhw}); - y_weight_indices.push_back(nhw); - - dfor(m_channel, m_y_corners.size())([&](auto c, auto) { - inner_indices.push_back(nhw); - auto nc = info.add_literal(migraphx::literal{m_nc_shape, {n, c}}); - nc_values.insert(nc_values.end(), {nc, nc, nc, nc}); - }); - }); - - auto inner_indices_t = concat_on_first_dim(info, inner_indices); - inner_indices_t = info.add_instruction( - make_op("reshape", - {{"dims", - {inner_indices_t->get_shape().elements() / nhw_shape.elements(), - nhw_shape.elements()}}}), - inner_indices_t); - std::array inner_y_samples; - std::transform( - m_y_corners.begin(), m_y_corners.end(), inner_y_samples.begin(), [&](auto corner) { - auto sample = info.add_instruction(make_op("gathernd"), corner, inner_indices_t); - return info.add_instruction( - make_op("reshape", {{"dims", {sample->get_shape().elements(), 1}}}), sample); - }); - - auto inner_y_t = concat_on_dim(info, inner_y_samples, 1); - auto elements = inner_y_t->get_shape().elements(); - inner_y_t = - info.add_instruction(make_op("reshape", {{"dims", {elements / 16, 4, 4}}}), inner_y_t); - inner_y_t = - info.add_instruction(make_op("transpose", {{"permutation", {0, 2, 1}}}), inner_y_t); - inner_y_t = info.add_instruction(make_op("reshape", {{"dims", {elements}}}), inner_y_t); - - std::array inner_x_samples; - std::transform( - m_x_corners.begin(), m_x_corners.end(), inner_x_samples.begin(), [&](auto corner) { - auto sample = info.add_instruction(make_op("gathernd"), corner, inner_indices_t); - return info.add_instruction( - make_op("reshape", {{"dims", {sample->get_shape().elements(), 1}}}), sample); - }); - - auto inner_x_t = concat_on_dim(info, inner_x_samples, 1); - inner_x_t = info.add_instruction( - make_op("reshape", {{"dims", {inner_x_t->get_shape().elements()}}}), inner_x_t); - - auto validate_index = [&](auto& index, auto& max) { - auto clip = info.add_common_op("clip", index, m_zero_l, max); - auto validation = info.add_common_op("equal", index, clip); - index = clip; - return validation; - }; - - auto y_validation = validate_index(inner_y_t, m_height_max_l); - auto x_validation = validate_index(inner_x_t, m_width_max_l); - - inner_y_t = info.add_instruction( - make_op("reshape", {{"dims", {inner_y_t->get_shape().elements(), 1}}}), inner_y_t); - inner_x_t = info.add_instruction( - make_op("reshape", {{"dims", {inner_x_t->get_shape().elements(), 1}}}), inner_x_t); - - auto nc_t = concat_on_first_dim(info, nc_values); - - auto indices_t = - info.add_instruction(make_op("concat", {{"axis", 1}}), inner_y_t, inner_x_t); - indices_t = info.add_instruction(make_op("concat", {{"axis", 1}}), nc_t, indices_t); - - auto samples = info.add_instruction(make_op("gathernd"), m_input, indices_t); - auto validation_t = info.add_common_op("logical_and", y_validation, x_validation); - samples = info.add_common_op("where", validation_t, samples, m_zero_l); - - auto x_weights_t = compute_weights( - info, x_weight_indices, m_x_weights, samples->get_shape().lens(), nhw_shape.elements()); - auto weighted_samples = info.add_common_op("mul", samples, x_weights_t); - weighted_samples = info.add_instruction( - make_op("reshape", {{"dims", {weighted_samples->get_shape().elements() / 4, 4}}}), - weighted_samples); - - auto coefficients = - info.add_instruction(make_op("reduce_sum", {{"axes", {1}}}), weighted_samples); - coefficients = info.add_instruction(make_op("squeeze", {{"axes", {1}}}), coefficients); - - auto y_weights_t = compute_weights(info, - y_weight_indices, - m_y_weights, - coefficients->get_shape().lens(), - nhw_shape.elements()); - auto weighted_coefficients = info.add_common_op("mul", coefficients, y_weights_t); - weighted_coefficients = info.add_instruction( - make_op("reshape", {{"dims", {weighted_coefficients->get_shape().elements() / 4, 4}}}), - weighted_coefficients); - - auto res = - info.add_instruction(make_op("reduce_sum", {{"axes", {1}}}), weighted_coefficients); - auto expected_shape = migraphx::shape{migraphx::shape::int64_type, - {m_batch, m_out_height, m_out_width, m_channel}}; - - res = info.add_instruction( - make_op("reshape", {{"dims", {m_batch, m_out_height, m_out_width, m_channel}}}), res); - res = info.add_instruction(make_op("transpose", {{"permutation", {0, 3, 1, 2}}}), res); - res = info.add_instruction( - make_op("convert", {{"target_type", m_input->get_shape().type()}}), res); - - return res; - } -}; - -struct parse_gridsample : op_parser -{ - std::vector operators() const { return {{"GridSample"}}; } - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - bool align_corners = false; - // Note: defult mode can be linear or bilinear depending on the onnx version - std::string mode = "linear"; - std::string padding_mode = "zeros"; - - if(contains(info.attributes, "align_corners")) - { - align_corners = parser.parse_value(info.attributes.at("align_corners")).at(); - } - - if(contains(info.attributes, "mode")) - { - mode = info.attributes.at("mode").s(); - } - - if(contains(info.attributes, "padding_mode")) - { - padding_mode = info.attributes.at("padding_mode").s(); - } - - const auto& grid = args.at(1); - const auto& grid_shape = grid->get_shape(); - if(not is_type_float(grid_shape.type())) - { - MIGRAPHX_THROW("PARSE_GRID_SAMPLE: grid input must have floating type"); - } - const auto& x = args.at(0); - const auto& x_dims = x->get_shape().lens().size(); - if(grid_shape.lens().size() != x_dims) - { - MIGRAPHX_THROW( - "PARSE_GRID_SAMPLE: x and grid inputs must have same number of dimensions"); - } - if(x_dims != 4) - { - MIGRAPHX_THROW("PARSE_GRID_SAMPLE: only 4-D inputs are supported"); - } - - return contains(mode, "nearest") - ? nearest_sampler(x, grid, align_corners, std::move(padding_mode), info) - .sample(info) - : (contains(mode, "linear") - ? linear_sampler(x, grid, align_corners, std::move(padding_mode), info) - .sample(info) - : bicubic_sampler(x, grid, align_corners, std::move(padding_mode), info) - .sample(info)); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_group_query_attention.cpp b/docker/rocm/migraphx/onnx/parse_group_query_attention.cpp deleted file mode 100644 index 7c36c252e..000000000 --- a/docker/rocm/migraphx/onnx/parse_group_query_attention.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_group_query_attention : op_parser -{ - std::vector operators() const { return {{"GroupQueryAttention"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - bool do_rotary = false; - std::size_t kv_num_heads = 0; - int local_window_size = -1; - std::size_t num_heads = 1; - bool rotary_interleaved = false; - float scale = 0.0; - if(contains(info.attributes, "do_rotary")) - { - do_rotary = parser.parse_value(info.attributes.at("do_rotary")).at(); - } - if(contains(info.attributes, "kv_num_heads")) - { - kv_num_heads = parser.parse_value(info.attributes.at("kv_num_heads")).at(); - } - if(contains(info.attributes, "local_window_size")) - { - local_window_size = - parser.parse_value(info.attributes.at("local_window_size")).at(); - } - if(contains(info.attributes, "num_heads")) - { - num_heads = parser.parse_value(info.attributes.at("num_heads")).at(); - } - if(contains(info.attributes, "rotary_interleaved")) - { - rotary_interleaved = - parser.parse_value(info.attributes.at("rotary_interleaved")).at(); - } - if(contains(info.attributes, "scale")) - { - scale = parser.parse_value(info.attributes.at("scale")).at(); - } - - if(args.size() < 7 or args.size() > 9) - { - MIGRAPHX_THROW("GroupQueryAttention: Wrong number of inputs provided"); - } - - auto gqa = info.add_instruction(make_op("group_query_attention", - {{"do_rotary", do_rotary}, - {"kv_num_heads", kv_num_heads}, - {"local_window_size", local_window_size}, - {"num_heads", num_heads}, - {"rotary_interleaved", rotary_interleaved}, - {"scale", scale}}), - args); - auto gqa_output = info.add_instruction(make_op("get_tuple_elem", {{"index", 0}}), gqa); - auto gqa_present_key = info.add_instruction(make_op("get_tuple_elem", {{"index", 1}}), gqa); - auto gqa_present_value = - info.add_instruction(make_op("get_tuple_elem", {{"index", 2}}), gqa); - - return {gqa_output, gqa_present_key, gqa_present_value}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_groupnorm.cpp b/docker/rocm/migraphx/onnx/parse_groupnorm.cpp deleted file mode 100644 index 770537fc8..000000000 --- a/docker/rocm/migraphx/onnx/parse_groupnorm.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_groupnorm : op_parser -{ - std::vector operators() const { return {{"GroupNormalization"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - float epsilon = 1e-5f; - if(contains(info.attributes, "epsilon")) - { - epsilon = parser.parse_value(info.attributes.at("epsilon")).at(); - } - size_t num_groups; - if(contains(info.attributes, "num_groups")) - { - num_groups = parser.parse_value(info.attributes.at("num_groups")).at(); - } - else - { - MIGRAPHX_THROW("PARSE_GROUPNORM: num_groups must be available"); - } - - if(args.size() != 3) - { - MIGRAPHX_THROW("PARSE_GROUPNORM: invalid input count"); - } - - auto x = args.at(0); - auto scale = args.at(1); - auto bias = args.at(2); - - auto x_shape = x->get_shape(); - auto x_dtype = x_shape.type(); - auto x_dims = x_shape.lens(); - - if(x_shape.ndim() <= 2) - { - MIGRAPHX_THROW("PARSE_GROUPNORM: invalid input shape"); - } - - auto c = x_shape.lens().at(1); - if(c % num_groups != 0) - { - MIGRAPHX_THROW( - "PARSE_GROUPNORM: num_groups should be a divisor of the number of channels"); - } - auto group_size = c / num_groups; - if(scale->get_shape().ndim() != 1 or scale->get_shape().lens().at(0) != num_groups) - { - MIGRAPHX_THROW("PARSE_GROUPNORM: scale tensor shape should be num_groups"); - } - if(bias->get_shape().ndim() != 1 or bias->get_shape().lens().at(0) != num_groups) - { - MIGRAPHX_THROW("PARSE_GROUPNORM: bias tensor shape should be num_groups"); - } - - // Original shape: N x C x D1 x ... x Dn - // New shape: N x num_groups x C // num_groups x D1 x ... x Dn - - std::vector dims = {x_dims.at(0), num_groups, group_size}; - std::copy(x_dims.begin() + 2, x_dims.end(), std::back_inserter(dims)); - auto x_reshaped = info.add_instruction(make_op("reshape", {{"dims", dims}}), x); - - // Axes for D1 x ... x Dn - std::vector axes(dims.size() - 2); - std::iota(axes.begin(), axes.end(), 2); - - // y = (x - mean) * rsqrt(variance + epsilon) * scale + bias - // mean = reduce_mean({D1, D2, ... Dk}, x) - // variance = reduce_mean({D1, D2, ... Dk}, (x - mean)^2) - - auto mean = info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), x_reshaped); - auto x_sub_mean = info.add_common_op("sub", x_reshaped, mean); - auto x_sqdiff_mean = info.add_common_op("sqdiff", x_reshaped, mean); - auto variance = - info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), x_sqdiff_mean); - epsilon = - (x_dtype == migraphx::shape::half_type and std::abs(epsilon) < 1e-7) ? 1e-7 : epsilon; - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_dtype}, {epsilon}}); - auto var_eps = info.add_common_op("add", variance, eps); - auto rsqrt = info.add_instruction(make_op("rsqrt"), var_eps); - auto result = info.add_common_op("mul", x_sub_mean, rsqrt); - auto scale_bcast = - info.add_instruction(make_op("broadcast", {{"axis", 1}, {"out_lens", dims}}), scale); - auto bias_bcast = - info.add_instruction(make_op("broadcast", {{"axis", 1}, {"out_lens", dims}}), bias); - auto scaled = info.add_instruction(make_op("mul"), result, scale_bcast); - auto y = info.add_instruction(make_op("add"), scaled, bias_bcast); - return info.add_instruction(make_op("reshape", {{"dims", x_dims}}), y); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_gru.cpp b/docker/rocm/migraphx/onnx/parse_gru.cpp deleted file mode 100644 index 46cb3b7ba..000000000 --- a/docker/rocm/migraphx/onnx/parse_gru.cpp +++ /dev/null @@ -1,193 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void gru_transpose_inputs(onnx_parser::node_info& info, std::vector& args) -{ - std::vector perm{1, 0, 2}; - args[0] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[0]); - - if(not args[5]->is_undefined()) - { - args[5] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[5]); - } -} - -void gru_transpose_outputs(onnx_parser::node_info& info, - instruction_ref& hidden_states, - instruction_ref& last_output) -{ - std::vector perm_hs{2, 0, 1, 3}; - hidden_states = - info.add_instruction(make_op("transpose", {{"permutation", perm_hs}}), hidden_states); - std::vector perm_last{1, 0, 2}; - last_output = - info.add_instruction(make_op("transpose", {{"permutation", perm_last}}), last_output); -} - -struct parse_gru : op_parser -{ - std::vector operators() const { return {{"GRU"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - migraphx::shape input_shape = args[0]->get_shape(); - std::size_t hidden_size = args[2]->get_shape().lens()[2]; - - if(contains(info.attributes, "hidden_size")) - { - std::size_t hidden_size_att = - parser.parse_value(info.attributes.at("hidden_size")).at(); - if(hidden_size != hidden_size_att) - { - MIGRAPHX_THROW("GRU: hidden size mismatch in input and attribute"); - } - } - - // Handling of direction to be added later - std::string direction{"forward"}; - if(contains(info.attributes, "direction")) - { - direction = info.attributes.at("direction").s(); - } - - op::rnn_direction dirct = op::rnn_direction::forward; - if(direction == "bidirectional") - { - dirct = op::rnn_direction::bidirectional; - } - else if(direction == "reverse") - { - dirct = op::rnn_direction::reverse; - } - - // set default activation functions - std::vector vec_names = {"sigmoid", "tanh"}; - if(dirct == op::rnn_direction::bidirectional) - { - // repeat the activation functions - vec_names.push_back(vec_names.at(0)); - vec_names.push_back(vec_names.at(1)); - } - - if(contains(info.attributes, "activations")) - { - auto names = info.attributes.at("activations").strings(); - vec_names.clear(); - vec_names.resize(names.size()); - std::transform(names.begin(), names.end(), vec_names.begin(), [](auto name) { - return to_lower(name); - }); - } - - auto num_actv_functions = dirct == op::rnn_direction::bidirectional ? 4 : 2; - if(vec_names.size() != static_cast(num_actv_functions)) - { - MIGRAPHX_THROW("GRU: Invalid activation functions number, should be: " + - to_string(num_actv_functions)); - } - - auto name_it = std::find_if(vec_names.begin(), vec_names.end(), [&](auto& name) { - return (map_activation_functions().count(name) == 0); - }); - if(name_it != vec_names.end()) - { - MIGRAPHX_THROW("GRU: activation function " + std::string(*name_it) + " not supported"); - } - - std::vector vec_actv_funcs(vec_names.size()); - std::transform(vec_names.begin(), - vec_names.end(), - vec_actv_funcs.begin(), - [&](const auto& name) { return map_activation_functions().at(name); }); - - float clip = 0.0; - if(contains(info.attributes, "clip")) - { - clip = parser.parse_value(info.attributes.at("clip")).at(); - } - - int layout = 0; - if(contains(info.attributes, "layout")) - { - layout = parser.parse_value(info.attributes.at("layout")).at(); - } - - int linear_before_reset = 0; - if(contains(info.attributes, "linear_before_reset")) - { - linear_before_reset = - parser.parse_value(info.attributes.at("linear_before_reset")).at(); - } - - // append undefined opeator to make 6 arguments - if(args.size() < 6) - { - auto ins = info.add_instruction(make_op("undefined")); - args.insert(args.end(), 6 - args.size(), ins); - } - - if(layout != 0) - { - gru_transpose_inputs(info, args); - } - - // first output for concatenation of hidden states - auto hidden_states = - info.add_instruction(make_op("gru", - {{"hidden_size", hidden_size}, - {"actv_func", to_value(vec_actv_funcs)}, - {"direction", dirct}, - {"clip", clip}, - {"linear_before_reset", linear_before_reset}}), - args); - - // second output for last gru output - auto last_output = info.add_instruction(make_op("rnn_last_hs_output"), hidden_states); - - if(layout != 0) - { - gru_transpose_outputs(info, hidden_states, last_output); - } - - return {hidden_states, last_output}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_hardmax.cpp b/docker/rocm/migraphx/onnx/parse_hardmax.cpp deleted file mode 100644 index d21e4561d..000000000 --- a/docker/rocm/migraphx/onnx/parse_hardmax.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_hardmax : op_parser -{ - std::vector operators() const { return {{"Hardmax", "hardmax"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - auto input = args[0]; - auto input_lens = input->get_shape().lens(); - - // default axis value is -1 for opset 13 - int64_t axis = -1; - - // axis value is 1 for previous opset versions - if(parser.opset_version < 13) - { - axis = 1; - } - - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - - if(parser.opset_version < 13) - { - // input is coerced into a 2D matrix of size NxD - axis = axis < 0 ? axis + input_lens.size() : axis; - size_t n = 1; - for(int i = 0; i < axis; i++) - { - n *= input_lens[i]; - } - size_t d = input->get_shape().elements() / n; - - input = info.add_instruction(make_op("reshape", {{"dims", {n, d}}}), input); - axis = 1; - } - - auto input_type = input->get_shape().type(); - auto indices = info.add_instruction(make_op("argmax", {{"axis", axis}}), input); - auto data = info.add_instruction( - make_op("multibroadcast", {{"out_lens", input->get_shape().lens()}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {0}})); - auto updates = info.add_instruction( - make_op("multibroadcast", {{"out_lens", indices->get_shape().lens()}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {1}})); - auto output = - info.add_instruction(make_op("scatter_none", {{"axis", axis}}), data, indices, updates); - - if(parser.opset_version < 13) - { - output = info.add_instruction(make_op("reshape", {{"dims", input_lens}}), output); - } - - return output; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_hardsigmoid.cpp b/docker/rocm/migraphx/onnx/parse_hardsigmoid.cpp deleted file mode 100644 index b8043243f..000000000 --- a/docker/rocm/migraphx/onnx/parse_hardsigmoid.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_hardsigmoid : op_parser -{ - std::vector operators() const { return {{"HardSigmoid"}, {"HardSwish"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - float alpha = 0.2; - float beta = 0.5; - if(opd.onnx_name == "HardSwish") - { - alpha = 1.0 / 6.0; - } - else - { - if(contains(info.attributes, "alpha")) - alpha = info.attributes.at("alpha").f(); - - if(contains(info.attributes, "beta")) - beta = info.attributes.at("beta").f(); - } - - auto input_lens = args[0]->get_shape().lens(); - auto input_type = args[0]->get_shape().type(); - auto mb_alpha = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {alpha}})); - auto mb_beta = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {beta}})); - auto mb_zero = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {0}})); - auto mb_one = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {1}})); - - auto mul = info.add_instruction(migraphx::make_op("mul"), mb_alpha, args[0]); - auto add = info.add_instruction(migraphx::make_op("add"), mb_beta, mul); - auto hardsigmoid = info.add_instruction(migraphx::make_op("clip"), add, mb_zero, mb_one); - if(opd.onnx_name == "HardSwish") - return info.add_instruction(migraphx::make_op("mul"), args[0], hardsigmoid); - - return hardsigmoid; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_if.cpp b/docker/rocm/migraphx/onnx/parse_if.cpp deleted file mode 100644 index 5fa642b7a..000000000 --- a/docker/rocm/migraphx/onnx/parse_if.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_if : op_parser -{ - std::vector operators() const { return {{"If"}}; } - - std::vector parse(const op_desc& /*opd*/, - onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - const auto& then_graph = info.attributes.at("then_branch").g(); - const auto& else_graph = info.attributes.at("else_branch").g(); - - if(args.front()->get_shape().elements() != 1) - { - MIGRAPHX_THROW("PARSE_IF: " + info.name + - " condition input can have only one element!"); - } - - // Fold instruction if condition is constant thus can be evaled - // prior to inference - if(args.front()->can_eval()) - { - auto cond_arg = args.front()->eval(); - auto* mod = info.mod; - // then branch - if(cond_arg.at()) - { - return parser.parse_graph(mod, then_graph, true); - } - // else branch - else - { - return parser.parse_graph(mod, else_graph, true); - } - } - - std::string then_name = info.name + "_if"; - module_ref then_mdl = parser.prog.create_module(then_name); - - std::string else_name = info.name + "_else"; - module_ref else_mdl = parser.prog.create_module(else_name); - - // parse the then sub_graph - (void)parser.parse_graph(then_mdl, then_graph); - - // parse_the else sub_graph - (void)parser.parse_graph(else_mdl, else_graph); - - auto then_out_shapes = then_mdl->get_output_shapes(); - auto else_out_shapes = else_mdl->get_output_shapes(); - if(not std::equal(then_out_shapes.begin(), - then_out_shapes.end(), - else_out_shapes.begin(), - else_out_shapes.end())) - { - MIGRAPHX_THROW("PARSE_IF: " + info.name + - " then and else sub_grahps must have same output shapes!"); - } - - auto if_ret = info.add_instruction(make_op("if"), args, {then_mdl, else_mdl}); - auto out_s = if_ret->get_shape(); - assert(out_s.type() == shape::tuple_type); - - const auto& vec_shapes = out_s.sub_shapes(); - std::vector out_inss; - for(std::size_t i = 0; i < vec_shapes.size(); ++i) - { - auto ret = info.add_instruction(make_op("get_tuple_elem", {{"index", i}}), if_ret); - out_inss.push_back(ret); - } - - return out_inss; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_imagescalar.cpp b/docker/rocm/migraphx/onnx/parse_imagescalar.cpp deleted file mode 100644 index 7e0b9de63..000000000 --- a/docker/rocm/migraphx/onnx/parse_imagescalar.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_imagescalar : op_parser -{ - std::vector operators() const { return {{"ImageScaler"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - float scale = 1.0; - std::vector bias{}; - if(contains(info.attributes, "scale")) - { - scale = parser.parse_value(info.attributes.at("scale")).at(); - } - - if(contains(info.attributes, "bias")) - { - auto&& bias_floats = info.attributes["bias"].floats(); - bias = std::vector(bias_floats.begin(), bias_floats.end()); - } - auto input_shape = args.front()->get_shape(); - auto const& input_lens = input_shape.lens(); - auto input_type = input_shape.type(); - - auto scale_val = info.add_literal(literal{shape{input_type}, {scale}}); - auto bias_vals = info.add_literal(literal{shape{input_type, {bias.size()}}, bias}); - - auto scale_tensor = info.add_instruction( - migraphx::make_op("scalar", {{"scalar_bcst_dims", input_lens}}), scale_val); - auto img_scaled = - info.add_instruction(migraphx::make_op("mul"), args.front(), scale_tensor); - auto bias_bcast = info.add_instruction( - migraphx::make_op("broadcast", {{"axis", 1}, {"out_lens", input_lens}}), bias_vals); - return info.add_instruction(migraphx::make_op("add"), img_scaled, bias_bcast); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_instancenorm.cpp b/docker/rocm/migraphx/onnx/parse_instancenorm.cpp deleted file mode 100644 index 05a066e14..000000000 --- a/docker/rocm/migraphx/onnx/parse_instancenorm.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_FP16_INSTANCENORM_CONVERT); - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_instancenorm : op_parser -{ - std::set valid_types = { - shape::float_type, shape::half_type, shape::double_type, shape::bf16_type}; - - std::vector operators() const { return {{"InstanceNormalization"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector oargs) const - { - // y = scale * ( x - mean ) / sqrt ( variance + epsilon ) + bias - // mean = reduce_mean({D1, D2, ... Dk}, x) - // variance = reduce_mean({D1, D2, ... Dk}, (x - mean)^2) - // Convert fp16 to fp32 to workaround for FP16 accuracy issues with reduce_mean/variance. - bool convert_fp16 = true; - if(enabled(MIGRAPHX_DISABLE_FP16_INSTANCENORM_CONVERT{})) - { - convert_fp16 = false; - } - float epsilon = 1e-5f; - if(contains(info.attributes, "epsilon")) - { - epsilon = parser.parse_value(info.attributes.at("epsilon")).at(); - } - auto dtype = oargs[0]->get_shape().type(); - auto literal_dtype = dtype; - std::vector args; - // cppcheck-suppress knownConditionTrueFalse - if(dtype == shape::half_type and convert_fp16) - { - std::transform(oargs.begin(), oargs.end(), std::back_inserter(args), [&](const auto i) { - return info.add_instruction( - make_op("convert", {{"target_type", shape::float_type}}), i); - }); - literal_dtype = shape::float_type; - } - else - { - args = oargs; - } - - auto x = args[0]; - auto scale = args[1]; - auto bias = args[2]; - if(not contains(valid_types, dtype)) - MIGRAPHX_THROW(opd.onnx_name + ": invalid output type: " + std::to_string(dtype) + - ". Valid types are 1 (float), 10 (half), and 11 (double)."); - - auto ndims = x->get_shape().ndim(); - assert(ndims >= 2); - auto kdims = ndims - 2; - std::vector axes(kdims); - std::iota(axes.begin(), axes.end(), 2); - auto mean = info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), x); - - // Use add_common_op() to insert multibroadcast/convert instructions where needed when - // inputs may be either static or dynamic. - auto l1 = info.add_common_op("sub", x, mean); - // for the fp16, if not converting to fp32 then divide `x` and `mean` by `sqrt(n)` and take - // reduce_sum to calculate variance i.e. - // var = reduce_sum((x/s_n - mean/s_n)^2) where s_n = sqrt(n) - std::string reduce_op_name = - (dtype == shape::half_type and not convert_fp16) ? "reduce_sum" : "reduce_mean"; - if(dtype == shape::half_type and not convert_fp16) - { - if(x->get_shape().dynamic()) - { - MIGRAPHX_THROW("PARSE_INSTANCENORM: half type not supported with dynamic shape " - "unless convert_fp16 is TRUE"); - } - auto dims = x->get_shape().lens(); - double n = - std::accumulate(dims.begin() + 2, dims.end(), 1, [&](const auto& i, const auto& j) { - return i * j; - }); - n = 1.0 / std::sqrt(n); - auto n_literal = info.add_literal(literal{dtype, {n}}); - x = info.add_common_op("mul", {x, n_literal}); - } - auto l0 = info.add_common_op("sqdiff", x, mean); - auto variance = info.add_instruction(make_op(reduce_op_name, {{"axes", axes}}), l0); - auto epsilon_literal = info.add_literal(literal{shape{literal_dtype}, {epsilon}}); - auto l2 = info.add_common_op("add", variance, epsilon_literal); - - auto l3 = info.add_instruction(make_op("rsqrt"), l2); - auto l4 = info.add_common_op("mul", l1, l3); - - // add_common_op() doesn't apply the plain broadcast op, so we add that op explicitly for - // both scale and bias. - instruction_ref scale_bcast; - instruction_ref bias_bcast; - if(x->get_shape().dynamic()) - { - scale_bcast = info.add_instruction(make_op("broadcast", {{"axis", 1}}), scale, x); - bias_bcast = info.add_instruction(make_op("broadcast", {{"axis", 1}}), bias, x); - } - else - { - auto dims = x->get_shape().lens(); - scale_bcast = info.add_instruction( - make_op("broadcast", {{"axis", 1}, {"out_lens", dims}}), scale); - bias_bcast = - info.add_instruction(make_op("broadcast", {{"axis", 1}, {"out_lens", dims}}), bias); - } - auto l5 = info.add_instruction(make_op("mul"), l4, scale_bcast); - auto ret = info.add_instruction(make_op("add"), l5, bias_bcast); - if(dtype == shape::half_type and convert_fp16) - { - return info.add_instruction(make_op("convert", {{"target_type", shape::half_type}}), - ret); - } - return ret; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_isinf.cpp b/docker/rocm/migraphx/onnx/parse_isinf.cpp deleted file mode 100644 index a0557b1e4..000000000 --- a/docker/rocm/migraphx/onnx/parse_isinf.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_isinf : op_parser -{ - std::vector operators() const { return {{"IsInf", "isinf"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - const std::vector& args) const - { - bool detect_negative = true; - bool detect_positive = true; - if(contains(info.attributes, "detect_negative")) - { - detect_negative = static_cast( - parser.parse_value(info.attributes.at("detect_negative")).at()); - } - - if(contains(info.attributes, "detect_positive")) - { - detect_positive = static_cast( - parser.parse_value(info.attributes.at("detect_positive")).at()); - } - - auto x_shape = args[0]->get_shape(); - if(not detect_negative and not detect_positive) - { - return info.add_instruction( - make_op("multibroadcast", {{"out_lens", x_shape.lens()}}), - info.add_literal(migraphx::literal{migraphx::shape{shape::bool_type}, {false}})); - } - - auto is_inf = info.add_instruction(make_op("isinf"), args[0]); - if(detect_negative and detect_positive) - { - return is_inf; - } - - auto zero_l = info.add_literal(migraphx::literal{migraphx::shape{x_shape.type()}, {0}}); - auto mb_zero = - info.add_instruction(make_op("multibroadcast", {{"out_lens", x_shape.lens()}}), zero_l); - - auto cond = info.add_broadcastable_binary_op( - detect_negative ? "less" : "greater", args[0], mb_zero); - if(cond->get_shape().type() != shape::bool_type) - { - cond = - info.add_instruction(make_op("convert", {{"target_type", shape::bool_type}}), cond); - } - return info.add_instruction(make_op("logical_and"), is_inf, cond); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_layernorm.cpp b/docker/rocm/migraphx/onnx/parse_layernorm.cpp deleted file mode 100644 index 5e6e4da70..000000000 --- a/docker/rocm/migraphx/onnx/parse_layernorm.cpp +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_layernorm : op_parser -{ - std::vector operators() const { return {{"LayerNormalization"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - int64_t axis = -1; - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - float epsilon = 1e-5f; - if(contains(info.attributes, "epsilon")) - { - epsilon = parser.parse_value(info.attributes.at("epsilon")).at(); - } - if(contains(info.attributes, "stash_type")) - { - std::cerr << "WARNING: LAYERNORM does not support stash_type, it will be ignored.\n"; - } - - if(args.size() < 2 or args.size() > 3) - { - MIGRAPHX_THROW("PARSE_LAYERNORM: invalid input count"); - } - - auto x = args.at(0); - auto scale = args.at(1); - bool skip_bias = args.size() == 2; - instruction_ref bias; - if(not skip_bias) - { - bias = args.at(2); - } - - auto x_shape = x->get_shape(); - auto x_dtype = x_shape.type(); - int64_t x_rank = x_shape.ndim(); - - if(x_rank < 2) - { - MIGRAPHX_THROW("PARSE_LAYERNORM: invalid input shape"); - } - - // If rank(X) is r, axis' allowed range is [-r, r) - if(axis < -x_rank or axis >= x_rank) - { - MIGRAPHX_THROW("PARSE_LAYERNORM: invalid axis"); - } - - // y = (x - mean) * rsqrt(variance + epsilon) * scale + bias - // mean = reduce_mean({D1, D2, ... Dk}, x) - // variance = reduce_mean({D1, D2, ... Dk}, (x - mean)^2) - - // axis can be negative - axis = axis < 0 ? axis + x_rank : axis; - - auto kdims = x_rank - axis; - std::vector axes(kdims); - std::iota(axes.begin(), axes.end(), axis); - auto skipped_axes = x_rank - kdims; - - auto mean = info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), x); - auto x_sub_mean = info.add_common_op("sub", x, mean); - auto x_sqdiff_mean = info.add_common_op("sqdiff", x, mean); - auto variance = - info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), x_sqdiff_mean); - epsilon = - (x_dtype == migraphx::shape::half_type and std::abs(epsilon) < 1e-7) ? 1e-7 : epsilon; - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_dtype}, {epsilon}}); - auto var_eps = info.add_common_op("add", variance, eps); - auto rsqrt = info.add_instruction(make_op("rsqrt"), var_eps); - auto result = info.add_common_op("mul", x_sub_mean, rsqrt); - - instruction_ref scale_bcast = scale; - instruction_ref bias_bcast = bias; - if(skipped_axes > 0) - { - auto x_dims = x_shape.lens(); - scale_bcast = info.add_instruction( - make_op("broadcast", {{"axis", skipped_axes}, {"out_lens", x_dims}}), scale); - if(not skip_bias) - { - bias_bcast = info.add_instruction( - make_op("broadcast", {{"axis", skipped_axes}, {"out_lens", x_dims}}), bias); - } - } - auto scaled = info.add_instruction(make_op("mul"), result, scale_bcast); - auto y = skip_bias ? scaled : info.add_instruction(make_op("add"), scaled, bias_bcast); - return {y, mean, rsqrt}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_lessorequal.cpp b/docker/rocm/migraphx/onnx/parse_lessorequal.cpp deleted file mode 100644 index 4d36710c6..000000000 --- a/docker/rocm/migraphx/onnx/parse_lessorequal.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_lessorequal : op_parser -{ - std::vector operators() const { return {{"LessOrEqual"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto in_res = info.add_broadcastable_binary_op("greater", args[0], args[1]); - if(in_res->get_shape().type() != shape::bool_type) - { - in_res = info.add_instruction(make_op("convert", {{"target_type", shape::bool_type}}), - in_res); - } - return info.add_instruction(make_op("not"), in_res); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_loop.cpp b/docker/rocm/migraphx/onnx/parse_loop.cpp deleted file mode 100644 index 742e475cd..000000000 --- a/docker/rocm/migraphx/onnx/parse_loop.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_loop : op_parser -{ - std::vector operators() const { return {{"Loop"}}; } - - std::vector parse(const op_desc& /*opd*/, - onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - // default value of the max_iter_num - int64_t max_iterations = parser.max_loop_iterations; - // iteration input is empty - if(args.at(0)->name() == "undefined") - { - shape iter_s{shape::int64_type}; - args[0] = info.add_literal(literal(iter_s, {max_iterations})); - } - else - { - auto arg_iters = args.at(0)->eval(); - if(not arg_iters.empty()) - { - max_iterations = arg_iters.at(); - } - } - - // cap max_iter because loop uses static shapes with max_iter size and huge numbers - // here can cause overflow - if(max_iterations > parser.limit_max_iterations) - { - std::cerr << "WARNING: PARSE_LOOP max_iterations exceeds the maximum loop " - "iterations limit, it will be changed from " - << max_iterations << " to " << parser.limit_max_iterations << ".\n"; - max_iterations = parser.limit_max_iterations; - } - - // condition input is empty - if(args.at(1)->name() == "undefined") - { - shape cond_s{shape::bool_type}; - args[1] = info.add_literal(literal(cond_s, {true})); - } - - // retrieve the subgraph - const auto& sub_graph = info.attributes.at("body").g(); - std::string mod_name = info.name + "_loop"; - module_ref sub_mod = parser.prog.create_module(mod_name); - - // parse the sub_graph - (void)parser.parse_graph(sub_mod, sub_graph); - - auto ret = info.add_instruction( - make_op("loop", {{"max_iterations", max_iterations}}), args, {sub_mod}); - auto out_s = ret->get_shape(); - assert(out_s.type() == shape::tuple_type); - - const auto& vec_shapes = out_s.sub_shapes(); - std::vector out_inss; - for(std::size_t i = 0; i < vec_shapes.size(); ++i) - { - auto r = info.add_instruction(make_op("get_tuple_elem", {{"index", i}}), ret); - out_inss.push_back(r); - } - - return out_inss; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_lpnormalization.cpp b/docker/rocm/migraphx/onnx/parse_lpnormalization.cpp deleted file mode 100644 index 919d66b33..000000000 --- a/docker/rocm/migraphx/onnx/parse_lpnormalization.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// Parser for LpNormalization ONNX operator. -/*! - Normalizes a tensor by the L1 or L2 norms along a given axis. - Norms that evaluate to 0 are changed to 1 to prevent division by zero. -*/ -struct parse_lpnormalization : op_parser -{ - std::vector operators() const { return {{"LpNormalization"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser&, - const onnx_parser::node_info& info, - std::vector args) const - { - int p = 2; - if(contains(info.attributes, "p")) - { - p = info.attributes.at("p").i(); - } - if(p != 1 and p != 2) - { - MIGRAPHX_THROW("LPNORMALIZATION: only L1 and L2 norm supported"); - } - auto input = args.front(); - auto input_shape = input->get_shape(); - const auto& input_lens = input_shape.lens(); - auto input_type = input_shape.type(); - std::ptrdiff_t num_axes = input_lens.size(); - std::ptrdiff_t axis = -1; - if(contains(info.attributes, "axis")) - { - axis = info.attributes.at("axis").i(); - if(axis < -num_axes or axis >= num_axes) - { - // handled in normalize_attributes but throwing here might be clearer - MIGRAPHX_THROW("LPNORMALIZATION: selected axis out of bounds"); - } - } - migraphx::instruction_ref p_val; - if(p == 1) - { - p_val = info.add_instruction(migraphx::make_op("abs"), input); - } - else - { - p_val = info.add_instruction(migraphx::make_op("mul"), input, input); - } - - // need to check for zeros from lp norm to prevent division by zero - // change them to 1 for the element-wise division - auto norms = - info.add_instruction(migraphx::make_op("reduce_sum", {{"axes", {axis}}}), p_val); - if(p == 2) - { - norms = info.add_instruction(migraphx::make_op("sqrt"), norms); - } - // broadcast back to initial shape, negative axis option doesn't work with unidirectional - norms = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), norms); - auto zero_mb = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {0.}})); - auto one_mb = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", input_lens}}), - info.add_literal(migraphx::literal{migraphx::shape{input_type}, {1.}})); - auto is_zero = info.add_instruction(migraphx::make_op("equal"), norms, zero_mb); - auto norms_zeros_to_one = - info.add_instruction(migraphx::make_op("where"), is_zero, one_mb, norms); - return info.add_instruction(migraphx::make_op("div"), input, norms_zeros_to_one); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_lstm.cpp b/docker/rocm/migraphx/onnx/parse_lstm.cpp deleted file mode 100644 index 60f13cd4b..000000000 --- a/docker/rocm/migraphx/onnx/parse_lstm.cpp +++ /dev/null @@ -1,216 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void lstm_actv_functions(op::rnn_direction dirct, std::vector& actv_func_names) -{ - // need 6 activation functions for bidirectional directions - if(dirct == op::rnn_direction::bidirectional) - { - actv_func_names.push_back(actv_func_names.at(0)); - actv_func_names.push_back(actv_func_names.at(1)); - actv_func_names.push_back(actv_func_names.at(2)); - } -} - -void lstm_transpose_inputs(onnx_parser::node_info& info, std::vector& args) -{ - std::vector perm{1, 0, 2}; - args[0] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[0]); - - if(args.size() >= 6 and not args[5]->is_undefined()) - { - args[5] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[5]); - } - - if(args.size() >= 7 and not args[6]->is_undefined()) - { - args[6] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[6]); - } -} - -void lstm_transpose_outputs(onnx_parser::node_info& info, - instruction_ref& hidden_states, - instruction_ref& last_output, - instruction_ref& last_cell_output) -{ - std::vector perm_hs{2, 0, 1, 3}; - hidden_states = - info.add_instruction(make_op("transpose", {{"permutation", perm_hs}}), hidden_states); - std::vector perm_last{1, 0, 2}; - last_output = - info.add_instruction(make_op("transpose", {{"permutation", perm_last}}), last_output); - last_cell_output = - info.add_instruction(make_op("transpose", {{"permutation", perm_last}}), last_cell_output); -} - -struct parse_lstm : op_parser -{ - std::vector operators() const { return {{"LSTM"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - migraphx::shape input_shape = args[0]->get_shape(); - std::size_t hidden_size = args[2]->get_shape().lens()[2]; - - if(contains(info.attributes, "hidden_size")) - { - std::size_t hidden_size_att = - parser.parse_value(info.attributes.at("hidden_size")).at(); - if(hidden_size != hidden_size_att) - { - MIGRAPHX_THROW("LSTM: hidden size mismatch in input and attribute"); - } - } - - // Handling of direction to be added later - std::string direction{"forward"}; - if(contains(info.attributes, "direction")) - { - direction = info.attributes.at("direction").s(); - } - - op::rnn_direction dirct = op::rnn_direction::forward; - if(direction == "bidirectional") - { - dirct = op::rnn_direction::bidirectional; - } - else if(direction == "reverse") - { - dirct = op::rnn_direction::reverse; - } - else if(direction == "forward") - { - dirct = op::rnn_direction::forward; - } - else - { - MIGRAPHX_THROW("LSTM: incorrect direction attribute"); - } - - // set default activation functions - std::vector vec_names = {"sigmoid", "tanh", "tanh"}; - lstm_actv_functions(dirct, vec_names); - - if(contains(info.attributes, "activations")) - { - auto names = info.attributes.at("activations").strings(); - vec_names.clear(); - vec_names.resize(names.size()); - std::transform(names.begin(), names.end(), vec_names.begin(), [](auto name) { - return to_lower(name); - }); - } - - auto num_actv_functions = dirct == op::rnn_direction::bidirectional ? 6 : 3; - if(vec_names.size() != static_cast(num_actv_functions)) - { - MIGRAPHX_THROW("LSTM: Invalid activation functions number, should be: " + - to_string(num_actv_functions)); - } - - auto name_it = std::find_if(vec_names.begin(), vec_names.end(), [&](auto& name) { - return (map_activation_functions().count(name) == 0); - }); - if(name_it != vec_names.end()) - { - MIGRAPHX_THROW("LSTM: activation function " + std::string(*name_it) + " not supported"); - } - - std::vector vec_actv_funcs(vec_names.size()); - std::transform(vec_names.begin(), - vec_names.end(), - vec_actv_funcs.begin(), - [&](const auto& name) { return map_activation_functions().at(name); }); - - float clip = 0.0; - if(contains(info.attributes, "clip")) - { - clip = parser.parse_value(info.attributes.at("clip")).at(); - } - - int input_forget = 0; - if(contains(info.attributes, "input_forget")) - { - input_forget = parser.parse_value(info.attributes.at("input_forget")).at(); - } - - int layout = 0; - if(contains(info.attributes, "layout")) - { - layout = parser.parse_value(info.attributes.at("layout")).at(); - } - - // append undefined opeator to make 6 arguments - if(args.size() < 8) - { - auto ins = info.add_instruction(make_op("undefined")); - args.insert(args.end(), 8 - args.size(), ins); - } - - if(layout != 0) - { - lstm_transpose_inputs(info, args); - } - - // first output for concatenation of hidden states - auto hidden_states = info.add_instruction(make_op("lstm", - {{"hidden_size", hidden_size}, - {"actv_func", to_value(vec_actv_funcs)}, - {"direction", dirct}, - {"clip", clip}, - {"input_forget", input_forget}}), - args); - - auto last_output = info.add_instruction(make_op("rnn_last_hs_output"), hidden_states); - - // third output for last cell output - auto last_cell_output = - info.add_instruction(make_op("rnn_last_cell_output"), hidden_states); - - if(layout != 0) - { - lstm_transpose_outputs(info, hidden_states, last_output, last_cell_output); - } - - return {hidden_states, last_output, last_cell_output}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_matmul.cpp b/docker/rocm/migraphx/onnx/parse_matmul.cpp deleted file mode 100644 index 302dc6b94..000000000 --- a/docker/rocm/migraphx/onnx/parse_matmul.cpp +++ /dev/null @@ -1,491 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_matmul : op_parser -{ - std::vector operators() const - { - return {{"MatMul", "dot"}, - {"MatMulInteger", "quant_dot"}, - {"MatMulIntegerToFloat", "quant_dot_scaled"}}; - } - - static void broadcast_dimensions(const onnx_parser::node_info& info, - const std::vector& s0_lens, - const std::vector& s1_lens, - const instruction_ref& a0, - const instruction_ref& a1, - instruction_ref& ba0, - instruction_ref& ba1) - { - // try broadcasting if dimensions other than last two do not match - if(not std::equal( - s0_lens.rbegin() + 2, s0_lens.rend(), s1_lens.rbegin() + 2, s1_lens.rend())) - { - auto l0_it = s0_lens.begin() + s0_lens.size() - 2; - std::vector l0_broadcasted_lens(s0_lens.begin(), l0_it); - auto l1_it = s1_lens.begin() + s1_lens.size() - 2; - std::vector l1_broadcasted_lens(s1_lens.begin(), l1_it); - auto output_lens = compute_broadcasted_lens(l0_broadcasted_lens, l1_broadcasted_lens); - l0_broadcasted_lens = output_lens; - l0_broadcasted_lens.insert(l0_broadcasted_lens.end(), l0_it, s0_lens.end()); - l1_broadcasted_lens = output_lens; - l1_broadcasted_lens.insert(l1_broadcasted_lens.end(), l1_it, s1_lens.end()); - if(s0_lens != l0_broadcasted_lens) - { - ba0 = info.add_instruction( - make_op("multibroadcast", {{"out_lens", l0_broadcasted_lens}}), a0); - } - if(s1_lens != l1_broadcasted_lens) - { - ba1 = info.add_instruction( - make_op("multibroadcast", {{"out_lens", l1_broadcasted_lens}}), a1); - } - } - } - - // Convert to half prior to a shift to ensure we preserve accuracy here then - // convert back to int8 - static instruction_ref add_int8_shift(const onnx_parser::node_info& info, - const instruction_ref& offset_op, - instruction_ref& unshifted_input) - { - auto unshifted_input_half = info.add_instruction( - migraphx::make_op("convert", {{"target_type", migraphx::shape::half_type}}), - unshifted_input); - - auto input_shifted_half = info.add_common_op("add", unshifted_input_half, offset_op); - - return info.add_instruction( - migraphx::make_op("convert", {{"target_type", migraphx::shape::int8_type}}), - input_shifted_half); - } - - static bool is_symmetric_zero_point(instruction_ref zp) - { - if(not zp->can_eval()) - return false; - - float check_value = 0; - if(zp->get_shape().type() == migraphx::shape::uint8_type) - check_value = 128; - - bool all_zeros = false; - zp->eval().visit([&](auto z) { - all_zeros = std::all_of( - z.begin(), z.end(), [&](auto val) { return float_equal(val, check_value); }); - }); - return all_zeros; - } - - static instruction_ref set_scale_arg(const onnx_parser::node_info& info, - const std::vector& args, - const instruction_ref& mat_input, - const int index) - { - instruction_ref scale_arg = args[index]; - std::set supported_dq_types = {migraphx::shape::float_type, - migraphx::shape::half_type}; - - auto scale_shape = scale_arg->get_shape(); - - if(not(contains(supported_dq_types, scale_shape.type()))) - { - MIGRAPHX_THROW("PARSE_QUANT_DOT_SCALED: Scales must be float or half_type"); - } - - if(scale_shape.lens().at(0) != *(mat_input->get_shape().lens().rbegin()) and - not scale_shape.scalar()) - { - MIGRAPHX_THROW("PARSE_QUANT_DOT_SCALED: Scale must have same dim as matrix column"); - } - - if(scale_shape.lens().size() > 1 and not scale_shape.scalar()) - { - MIGRAPHX_THROW("PARSE_QUANT_DOT_SCALED: Scales shape must be scalar or 1-D tensor"); - } - - if(scale_shape.scalar()) - { - scale_arg = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), scale_arg); - scale_shape = scale_arg->get_shape(); - } - - scale_arg = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), scale_arg); - - return scale_arg; - } - - static instruction_ref set_scale_bias(const std::vector& args, - const int index, - const migraphx::shape& scale_arg_shape, - const instruction_ref& compare_arg, - bool& has_valid_scale_bias) - { - has_valid_scale_bias = false; - - if(args.size() > index) - { - instruction_ref scale_bias_arg = args[index]; - std::set supported_dq_types = {migraphx::shape::float_type, - migraphx::shape::half_type}; - - if(not(contains(supported_dq_types, scale_bias_arg->get_shape().type()))) - { - MIGRAPHX_THROW("PARSE_QUANT_DOT_SCALED: Bias must be float or half_type"); - } - - if(scale_bias_arg->get_shape().type() != scale_arg_shape.type()) - { - MIGRAPHX_THROW("PARSE_QUANT_DOT_SCALED: Bias must be the same type as scales"); - } - - if(scale_bias_arg->get_shape().lens().at(0) != - *(compare_arg->get_shape().lens().rbegin())) - { - MIGRAPHX_THROW("PARSE_QUANT_DOT_SCALED: Bias have same dim as matrix B column"); - } - - has_valid_scale_bias = true; - return scale_bias_arg; - } - return compare_arg; - } - - static instruction_ref set_bias_arg(const std::string& name, - const std::vector& args, - const int index, - const instruction_ref& input, - bool& has_valid_bias) - { - has_valid_bias = false; - - if(args.size() > index) - { - instruction_ref bias_arg = args[index]; - if(bias_arg->get_shape().type() != input->get_shape().type()) - { - MIGRAPHX_THROW(name + ": zero point must be the same type as data"); - } - - // Don't return zero point if it will cause symmetric zero point. No need to bias - if(is_symmetric_zero_point(bias_arg)) - return input; - - has_valid_bias = true; - return bias_arg; - } - return input; - } - - static void shift_input_and_bias(const onnx_parser::node_info& info, - const instruction_ref& offset_op, - const bool has_bias, - instruction_ref& input, - instruction_ref& input_bias) - { - input = add_int8_shift(info, offset_op, input); - if(has_bias) - { - input_bias = add_int8_shift(info, offset_op, input_bias); - } - else - { - input_bias = input; - } - } - - static void handle_scaled_transposes(const onnx_parser::node_info& info, - instruction_ref& scale, - instruction_ref& zp, - bool no_zp) - { - if(no_zp) - { - scale = info.add_instruction(make_op("transpose", {{"permutation", {0, 1}}}), scale); - } - else - { - scale = info.add_instruction(make_op("transpose", {{"permutation", {0, 1}}}), scale); - zp = info.add_instruction(make_op("transpose", {{"permutation", {1, 0}}}), zp); - } - } - - static instruction_ref handle_dequantized(const onnx_parser::node_info& info, - const instruction_ref& a0, - const instruction_ref& scale_a0, - const instruction_ref& zp_a0, - bool no_zp) - { - instruction_ref dequantized_op; - - if(no_zp) - { - auto bc_scale_a0 = info.add_instruction( - make_op("multibroadcast", {{"out_lens", a0->get_shape().lens()}}), scale_a0); - dequantized_op = info.add_instruction(make_op("dequantizelinear"), a0, bc_scale_a0); - } - else - { - auto bc_scale_a0 = info.add_instruction( - make_op("multibroadcast", {{"out_lens", a0->get_shape().lens()}}), scale_a0); - - auto bc_zp_a0 = info.add_instruction( - make_op("multibroadcast", {{"out_lens", a0->get_shape().lens()}}), zp_a0); - - dequantized_op = - info.add_instruction(make_op("dequantizelinear"), a0, bc_scale_a0, bc_zp_a0); - } - return dequantized_op; - } - - static instruction_ref handle_scaled_output(const onnx_parser::node_info& info, - const instruction_ref& a0, - const instruction_ref& a1, - const instruction_ref& scale_a0, - const instruction_ref& scale_a1, - const instruction_ref& zp_a0, - const instruction_ref& zp_a1, - const instruction_ref& scaled_bias, - const bool has_scale_bias) - { - - instruction_ref unsq_zp_a0; - instruction_ref unsq_zp_a1; - - bool a0_has_no_zp = (a0 == zp_a0); - bool a1_has_no_zp = (a1 == zp_a1); - - if(not a0_has_no_zp) - { - unsq_zp_a0 = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), zp_a0); - if(zp_a0->get_shape().scalar()) - { - unsq_zp_a0 = - info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), unsq_zp_a0); - } - } - - if(not a1_has_no_zp) - { - unsq_zp_a1 = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), zp_a1); - if(zp_a1->get_shape().scalar()) - { - unsq_zp_a1 = - info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), unsq_zp_a1); - } - } - - auto dq_a0 = handle_dequantized(info, a0, scale_a0, unsq_zp_a0, a0_has_no_zp); - auto dq_a1 = handle_dequantized(info, a1, scale_a1, unsq_zp_a1, a1_has_no_zp); - auto res = info.add_instruction(make_op("dot"), dq_a0, dq_a1); - - // Handle case of the bias after scaling - if(has_scale_bias) - res = info.add_common_op("sub", res, scaled_bias); - - return res; - } - - static void handle_uint8_input(const onnx_parser::node_info& info, - const bool has_bias, - const instruction_ref& offset_op, - instruction_ref& arg, - instruction_ref& bias_arg) - { - auto arg_type = arg->get_shape().type(); - // always convert uint8 to int8 to avoid rollover - if(arg_type == migraphx::shape::uint8_type) - { - shift_input_and_bias(info, offset_op, has_bias, arg, bias_arg); - } - - // subtract bias from result after conversion - if(has_bias) - { - bias_arg = info.add_common_op("sub", arg, bias_arg); - } - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - std::string op_name{opd.op_name}; - auto a0 = args[0]; - auto a1 = args[1]; - auto s0 = a0->get_shape(); - auto s1 = a1->get_shape(); - - instruction_ref dot_res; - bool is_a_prepended = false; - bool is_b_appended = false; - if(s0.ndim() == 1) - { - is_a_prepended = true; - a0 = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), args[0]); - } - if(s1.ndim() == 1) - { - is_b_appended = true; - a1 = info.add_instruction(make_op("unsqueeze", {{"axes", {1}}}), args[1]); - } - - auto is_quant_dot = opd.op_name == "quant_dot"; - auto is_quant_dot_scaled = opd.op_name == "quant_dot_scaled"; - auto is_dot = opd.op_name == "dot"; - - if(s0.dynamic() or s1.dynamic()) - { - if(is_quant_dot or is_quant_dot_scaled) - { - MIGRAPHX_THROW(op_name + ": dynamic inputs not supported"); - } - - auto s0_dds = a0->get_shape().to_dynamic().dyn_dims(); - auto s1_dds = a1->get_shape().to_dynamic().dyn_dims(); - - if(not std::equal( - s0_dds.rbegin() + 2, s0_dds.rend(), s1_dds.rbegin() + 2, s1_dds.rend())) - { - auto broadcasted_a0 = info.add_instruction(make_op("broadcast_for_dot"), a0, a1); - auto broadcasted_a1 = info.add_instruction(make_op("broadcast_for_dot"), a1, a0); - dot_res = - info.add_instruction(make_op(opd.op_name), broadcasted_a0, broadcasted_a1); - } - else - { - dot_res = info.add_instruction(make_op(opd.op_name), a0, a1); - } - } - else - { - auto s0_lens = a0->get_shape().lens(); - auto s1_lens = a1->get_shape().lens(); - - if(is_dot and args.size() > 2) - { - MIGRAPHX_THROW(op_name + ": Bias Args not supported"); - } - - bool has_ba0 = false; - bool has_ba1 = false; - bool has_scale_bias = false; - - int a0_zp_index = 2; - int a1_zp_index = 3; - - instruction_ref scale_a0; - instruction_ref scale_a1; - // Handles case with for when scales are present in operator - if(is_quant_dot_scaled) - { - a0_zp_index = 4; - a1_zp_index = 5; - scale_a0 = set_scale_arg(info, args, a0, 2); - scale_a1 = set_scale_arg(info, args, a1, 3); - if(scale_a0->get_shape().type() != scale_a1->get_shape().type()) - { - MIGRAPHX_THROW(op_name + ": Scales must be the same type"); - } - } - - instruction_ref ba0 = set_bias_arg(op_name, args, a0_zp_index, a0, has_ba0); - instruction_ref ba1 = set_bias_arg(op_name, args, a1_zp_index, a1, has_ba1); - - // handle optional bias arg to the result - instruction_ref scaled_bias; - if(is_quant_dot_scaled) - { - auto scaled_index = 6; - scaled_bias = - set_scale_bias(args, scaled_index, scale_a1->get_shape(), a1, has_scale_bias); - } - - // Only INT8 or UINT8 type currently supported - std::set supported_types = {migraphx::shape::uint8_type, - migraphx::shape::int8_type}; - const auto a0_type = a0->get_shape().type(); - const auto a1_type = a1->get_shape().type(); - - if((not is_dot) and - (not contains(supported_types, a0_type) or not contains(supported_types, a1_type))) - { - MIGRAPHX_THROW(op_name + ": Unsupported type"); - } - - if((is_quant_dot and ((a0_type == migraphx::shape::uint8_type) or - (a1_type == migraphx::shape::uint8_type)))) - { - auto offset_op = info.add_literal( - migraphx::literal{migraphx::shape{migraphx::shape::half_type}, {-128}}); - handle_uint8_input(info, has_ba0, offset_op, a0, ba0); - handle_uint8_input(info, has_ba1, offset_op, a1, ba1); - } - - broadcast_dimensions(info, s0_lens, s1_lens, a0, a1, ba0, ba1); - - // Apply the scale to dequantize input to then perform a simple dot - // after the zero points are applied otherwise get a int32 output from the quantized - // equivalent. Ensure these are broadcasted accordingly before we perform a dot - if(is_quant_dot_scaled) - { - dot_res = handle_scaled_output( - info, a0, a1, scale_a0, scale_a1, ba0, ba1, scaled_bias, has_scale_bias); - } - else - { - dot_res = info.add_instruction(make_op(opd.op_name), ba0, ba1); - } - } - - // squeeze the appended or prepended dimensions - int64_t num_axis = dot_res->get_shape().ndim(); - if(is_a_prepended) - { - dot_res = info.add_instruction(make_op("squeeze", {{"axes", {num_axis - 2}}}), dot_res); - --num_axis; - } - if(is_b_appended) - { - dot_res = info.add_instruction(make_op("squeeze", {{"axes", {num_axis - 1}}}), dot_res); - } - - return dot_res; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_matmulnbits.cpp b/docker/rocm/migraphx/onnx/parse_matmulnbits.cpp deleted file mode 100644 index e5fb336df..000000000 --- a/docker/rocm/migraphx/onnx/parse_matmulnbits.cpp +++ /dev/null @@ -1,192 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "migraphx/errors.hpp" -#include "migraphx/instruction_ref.hpp" -#include "migraphx/onnx/onnx_parser.hpp" -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_matmulnbits : op_parser -{ - std::vector operators() const { return {{"MatMulNBits"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - const std::vector& args) const - { - const size_t n = parse_attribute(parser, info, "N"); - const size_t k = parse_attribute(parser, info, "K"); - const size_t bits = parse_attribute(parser, info, "bits"); - const size_t block_size = parse_attribute(parser, info, "block_size"); - - if(bits != 4) - MIGRAPHX_THROW("MatMulNBits: bits only supported for value of 4, actual value " + - std::to_string(bits)); - - if(block_size < 16 or (block_size & (block_size - 1)) != 0) - MIGRAPHX_THROW("MatMulNBits: block_size must be a power of 2 and >=16, actual value " + - std::to_string(block_size)); - - const size_t n_blocks_per_col = (k + block_size - 1) / block_size; - const size_t blob_size = std::ceil(block_size * bits / 8.0f); - - std::vector expected_b_lens{n, n_blocks_per_col, blob_size}; - if(args[1]->get_shape().lens() != expected_b_lens) - MIGRAPHX_THROW("MatMulNBits: Input B does not match expected dims: " + - to_string_range(expected_b_lens) + - ". Actual dims: " + to_string_range(args[1]->get_shape().lens())); - - const size_t expected_scales_lens = n * n_blocks_per_col; - if(args[2]->get_shape().elements() != expected_scales_lens) - MIGRAPHX_THROW("MatMulNBits: Input scales does not match expected dims: " + - to_string(expected_scales_lens) + - ". Actual dims: " + to_string_range(args[2]->get_shape().lens())); - - if(args.size() > 3) - { - std::vector expected_zp_lens{ - static_cast(n * std::ceil(n_blocks_per_col * bits / 8.0f))}; - if(args[3]->get_shape().lens() != expected_zp_lens) - MIGRAPHX_THROW("MatMulNBits: Input zero_points does not match expected dims: " + - to_string_range(expected_zp_lens) + - ". Actual dims: " + to_string_range(args[3]->get_shape().lens())); - } - - auto b = dequantize_b(info, n, k, block_size, args); - b = info.add_instruction(make_op("transpose", {{"permutation", {1, 0}}}), b); - return matmul(info, args[0], b); - } - - private: - int parse_attribute(const onnx_parser& parser, - onnx_parser::node_info& info, - const std::string& attribute_name) const - { - if(not contains(info.attributes, attribute_name)) - MIGRAPHX_THROW("MatMulNBits: Attribute " + attribute_name + - " required, but is missing"); - - return parser.parse_value(info.attributes[attribute_name]).at(); - } - - instruction_ref dequantize_b(onnx_parser::node_info& info, - int n, - int k, - int block_size, - const std::vector& args) const - { - auto b = unpack(info, n, k, args[1]); - - auto n_blocks_per_col = (k + block_size - 1) / block_size; - auto scales = info.add_instruction(make_op("reshape", {{"dims", {n, -1}}}), args[2]); - scales = prepare_blockwise_dq_arg(info, n, k, block_size, scales); - - instruction_ref zp; - if(args.size() == 4) - { - zp = unpack(info, n, n_blocks_per_col, args[3]); - zp = prepare_blockwise_dq_arg(info, n, k, block_size, zp); - } - else - { - zp = info.add_literal(literal{shape{shape::uint8_type, {1}}, {8}}); - zp = info.add_instruction( - make_op("multibroadcast", {{"out_lens", b->get_shape().lens()}}), zp); - } - return info.add_instruction(make_op("dequantizelinear"), {b, scales, zp}); - } - - instruction_ref unpack(onnx_parser::node_info& info, int n, int dim1, instruction_ref x) const - { - x = info.add_instruction(make_op("reshape", {{"dims", {n, -1}}}), x); - x = info.add_instruction(make_op("unpack_int4"), x); - if(x->get_shape().lens()[1] > dim1) - { - x = info.add_instruction( - make_op("slice", {{"axes", {1}}, {"starts", {0}}, {"ends", {dim1}}}), x); - } - return x; - } - - instruction_ref prepare_blockwise_dq_arg( - onnx_parser::node_info& info, int n, int k, int block_size, instruction_ref x) const - { - x = info.add_instruction(make_op("unsqueeze", {{"axes", {2}}}), x); - - auto bc_lens = x->get_shape().lens(); - bc_lens[2] = block_size; - x = info.add_instruction(make_op("multibroadcast", {{"out_lens", bc_lens}}), x); - x = info.add_instruction(make_op("reshape", {{"dims", {n, -1}}}), x); - - // Detect runt block - if(x->get_shape().lens()[1] > k) - { - x = info.add_instruction( - make_op("slice", {{"axes", {1}}, {"starts", {0}}, {"ends", {k}}}), x); - } - - return x; - } - - instruction_ref matmul(onnx_parser::node_info& info, instruction_ref a, instruction_ref b) const - { - const auto a_rank = a->get_shape().ndim(); - // B is always rank 2: - // If A is rank 1, unsqueeze A to make it rank 2 to prepare for dot - // If A is rank 2, just a regular dot - // If A is rank > 2, broadcast B to match outer dims of A to prepare for dot - if(a_rank == 1) - { - a = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), a); - } - else if(a_rank > 2) - { - auto b_lens = b->get_shape().lens(); - auto b_bc_lens = a->get_shape().lens(); - std::copy(b_lens.begin(), b_lens.end(), b_bc_lens.end() - 2); - b = info.add_instruction(make_op("multibroadcast", {{"out_lens", b_bc_lens}}), b); - } - - auto dot = info.add_instruction(make_op("dot"), a, b); - - if(a_rank == 1) - dot = info.add_instruction( - make_op("squeeze", {{"axes", {dot->get_shape().ndim() - 2}}}), dot); - - return dot; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_mean.cpp b/docker/rocm/migraphx/onnx/parse_mean.cpp deleted file mode 100644 index e6f422079..000000000 --- a/docker/rocm/migraphx/onnx/parse_mean.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_mean : op_parser -{ - std::set float_types = {shape::float_type, shape::half_type, shape::double_type}; - - std::vector operators() const { return {{"Mean"}}; } - - /// Calculates the element-wise mean of n>=1 input tensors - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto num_data = args.size(); - if(num_data == 1) - return args[0]; - - auto divisor = info.add_literal( - migraphx::literal{migraphx::shape{args[0]->get_shape().type()}, {num_data}}); - - if(contains(float_types, args[0]->get_shape().type())) - { - return std::accumulate(args.begin() + 1, - args.end(), - info.add_broadcastable_binary_op("div", args[0], divisor), - [&](auto mean, auto data_i) { - // Pre-divide each tensor element-wise by n to reduce risk of - // overflow during summation - auto div = - info.add_broadcastable_binary_op("div", data_i, divisor); - return info.add_broadcastable_binary_op("add", mean, div); - }); - } - else - { - // Compute sum before division for integral types - auto sum = std::accumulate( - args.begin() + 1, args.end(), args[0], [&](auto accum, auto data_i) { - return info.add_broadcastable_binary_op("add", accum, data_i); - }); - - return info.add_broadcastable_binary_op("div", sum, divisor); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_mean_variance_normalization.cpp b/docker/rocm/migraphx/onnx/parse_mean_variance_normalization.cpp deleted file mode 100644 index 75287d300..000000000 --- a/docker/rocm/migraphx/onnx/parse_mean_variance_normalization.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_mean_variance_normalization : op_parser -{ - std::vector operators() const { return {{"MeanVarianceNormalization"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - auto&& data = args.front(); - auto data_rank = data->get_shape().ndim(); - std::vector axes{0, 2, 3}; - - if(contains(info.attributes, "axes")) - { - const auto& axes_attr = info.attributes["axes"].ints(); - axes.assign(axes_attr.begin(), axes_attr.end()); - } - else if(data_rank != 4) - { - MIGRAPHX_THROW( - "Input tensor needs to be rank 4 when axes is not specified. Instead it is rank " + - std::to_string(data_rank)); - } - - if(axes.size() != data_rank - 1) - { - MIGRAPHX_THROW("Length of axes array needs to be equal to input tensor rank - 1"); - } - - auto data_mean = info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), data); - auto data_mean_squared = info.add_common_op("mul", data_mean, data_mean); - - auto data_squared = info.add_common_op("mul", data, data); - auto data_squared_mean = - info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), data_squared); - - auto mean_sub = info.add_common_op("sub", data_squared_mean, data_mean_squared); - auto std = info.add_common_op("sqrt", mean_sub); - - auto dividend = info.add_common_op("sub", data, data_mean); - auto epsilon = - info.add_literal({data->get_shape().type(), - {data->get_shape().type() == shape::half_type ? 1e-7 : 1e-9}}); - auto divisor = info.add_common_op("add", std, epsilon); - - return info.add_common_op("div", dividend, divisor); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_mod.cpp b/docker/rocm/migraphx/onnx/parse_mod.cpp deleted file mode 100644 index 33426d276..000000000 --- a/docker/rocm/migraphx/onnx/parse_mod.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_mod : op_parser -{ - std::vector operators() const { return {{"Mod"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - std::string mod = "mod"; - if(is_type_float(args[0]->get_shape().type()) or is_type_float(args[1]->get_shape().type())) - { - if(not contains(info.attributes, "fmod")) - { - MIGRAPHX_THROW("Mod operator with float args and fmod=0 invalid"); - } - } - - if(contains(info.attributes, "fmod")) - { - if(parser.parse_value(info.attributes.at("fmod")).at() == 1) - { - mod = "fmod"; - } - } - return info.add_common_op(mod, args[0], args[1]); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_multi_head_attention.cpp b/docker/rocm/migraphx/onnx/parse_multi_head_attention.cpp deleted file mode 100644 index 9bde20868..000000000 --- a/docker/rocm/migraphx/onnx/parse_multi_head_attention.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -enum class qkv_fomat_t -{ - q_k_v = 0, - q_k_v_cross = 1, - kv_packed = 2, - qkv_packed = 3 -}; - -struct multi_head_attention_parameters -{ - int64_t batch_size; - int64_t q_sequence_length; - int64_t kv_sequence_length; - int64_t hidden_size; - int64_t hidden_size_v; - int64_t head_size; - int64_t head_size_v; - qkv_fomat_t qkv_fomat; -}; - -struct parse_multi_head_attention : op_parser -{ - - std::vector operators() const { return {{"MultiHeadAttention"}}; } - - void unpack_qkv(const onnx_parser::node_info& info, - instruction_ref& query, - instruction_ref& key, - instruction_ref& value) const - { - // (batch_size, q_sequence_length, num_heads, 3, head_size) -> - // (3, batch_size, q_sequence_length, num_heads, head_size) - auto qkv_packed = - info.add_instruction(make_op("transpose", {{"permutation", {3, 0, 1, 2, 4}}}), query); - query = info.add_instruction( - make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), qkv_packed); - query = info.add_instruction(make_op("squeeze", {{"axes", {0}}}), query); - key = info.add_instruction( - make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), qkv_packed); - key = info.add_instruction(make_op("squeeze", {{"axes", {0}}}), key); - value = info.add_instruction( - make_op("slice", {{"axes", {0}}, {"starts", {2}}, {"ends", {3}}}), qkv_packed); - value = info.add_instruction(make_op("squeeze", {{"axes", {0}}}), value); - } - - void unpack_kv(const onnx_parser::node_info& info, - instruction_ref& key, - instruction_ref& value) const - { - // (batch_size, kv_sequence_length, num_heads, 2, head_size) -> - // (2, batch_size, kv_sequence_length, num_heads, head_size) - auto kv_packed = - info.add_instruction(make_op("transpose", {{"permutation", {3, 0, 1, 2, 4}}}), key); - key = info.add_instruction( - make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), kv_packed); - key = info.add_instruction(make_op("squeeze", {{"axes", {0}}}), key); - value = info.add_instruction( - make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), kv_packed); - value = info.add_instruction(make_op("squeeze", {{"axes", {0}}}), value); - } - - void check_inputs(const std::vector& args, - const int64_t num_heads, - multi_head_attention_parameters& params) const - { - if(args.empty() or args.size() > 3) - MIGRAPHX_THROW("MultiHeadAttention: Wrong number of inputs. Only 'query', 'key' and " - "'value' inputs are supported."); - - auto query_dim = args[0]->get_shape().ndim(); - auto query_lens = args[0]->get_shape().lens(); - - params.batch_size = query_lens[0]; - params.q_sequence_length = query_lens[1]; - - if(query_dim != 3 and query_dim != 5) - MIGRAPHX_THROW("MultiHeadAttention: Input 'query' rank needs to be 3 or 5, current: " + - std::to_string(query_dim)); - - if(query_dim == 5) - { - if(query_lens[2] != num_heads or query_lens[3] != 3) - MIGRAPHX_THROW("MultiHeadAttention: Input 'query' shape needs to be (batch_size, " - "q_sequence_length, num_heads, 3, head_size) for packed input."); - - params.kv_sequence_length = query_lens[1]; - params.head_size = query_lens[4]; - params.head_size_v = query_lens[4]; - params.hidden_size = num_heads * query_lens[4]; - params.hidden_size_v = num_heads * query_lens[4]; - params.qkv_fomat = qkv_fomat_t::qkv_packed; - } - else // query_dim == 3 - { - if(args.size() < 2) - MIGRAPHX_THROW("MultiHeadAttention: Wrong number of inputs, 'key' is missing."); - - params.hidden_size = query_lens[2]; - params.head_size = query_lens[2] / num_heads; - - auto key_dim = args[1]->get_shape().ndim(); - auto key_lens = args[1]->get_shape().lens(); - - if(key_dim < 3 or key_dim > 5) - MIGRAPHX_THROW( - "MultiHeadAttention: Input 'key' rank needs to be 3, 4 or 5, current: " + - std::to_string(key_dim)); - - if(key_dim == 5) - { - if(key_lens[0] != params.batch_size or key_lens[2] != num_heads or - key_lens[3] != 2 or key_lens[4] != params.head_size) - MIGRAPHX_THROW("MultiHeadAttention: Input 'key' shape needs to be (batch_size, " - "kv_sequence_length, num_heads, 2, head_size)"); - - params.kv_sequence_length = key_lens[1]; - params.hidden_size_v = params.hidden_size; - params.head_size_v = key_lens[4]; - params.qkv_fomat = qkv_fomat_t::kv_packed; - } - else - { - if(args.size() < 3) - MIGRAPHX_THROW( - "MultiHeadAttention: Wrong number of inputs, 'value' is missing."); - - auto value_dim = args[2]->get_shape().ndim(); - auto value_lens = args[2]->get_shape().lens(); - - if(key_dim != value_dim) - MIGRAPHX_THROW( - "MultiHeadAttention: Input 'key' and 'value' rank needs to be equal."); - - if(key_dim == 3) - { - if(key_lens[0] != params.batch_size or key_lens[2] != params.hidden_size) - MIGRAPHX_THROW("MultiHeadAttention: Input 'key' shape needs to be " - "(batch_size, kv_sequence_length, hidden_size)"); - - if(value_lens[0] != params.batch_size or value_lens[1] != key_lens[1]) - MIGRAPHX_THROW("MultiHeadAttention: Input 'value' shape needs to be " - "(batch_size, kv_sequence_length, hidden_size_v)"); - - params.kv_sequence_length = key_lens[1]; - params.hidden_size_v = value_lens[2]; - params.head_size_v = value_lens[2] / num_heads; - params.qkv_fomat = qkv_fomat_t::q_k_v; - } - else // key_dim == 4 - { - if(key_lens[0] != params.batch_size or key_lens[1] != num_heads or - key_lens[3] != params.head_size) - MIGRAPHX_THROW("MultiHeadAttention: Input 'key' shape needs to be " - "(batch_size, num_heads, kv_sequence_length, head_size)"); - - if(value_lens[0] != params.batch_size or value_lens[1] != num_heads or - value_lens[2] != key_lens[2]) - MIGRAPHX_THROW("MultiHeadAttention: Input 'value' shape needs to be " - "(batch_size, num_heads, kv_sequence_length, head_size_v)"); - - params.kv_sequence_length = key_lens[2]; - params.hidden_size_v = value_lens[3] * num_heads; - params.head_size_v = value_lens[3]; - params.qkv_fomat = qkv_fomat_t::q_k_v_cross; - } - } - } - } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - if(not contains(info.attributes, "num_heads")) - MIGRAPHX_THROW("MultiHeadAttention: num_heads attribute is required"); - - int64_t num_heads = parser.parse_value(info.attributes.at("num_heads")).at(); - - multi_head_attention_parameters params; - check_inputs(args, num_heads, params); - - auto query = args[0]; - instruction_ref key; - instruction_ref value; - - if(params.qkv_fomat == qkv_fomat_t::qkv_packed) - { - // Packed QKV: (batch_size, q_sequence_length, num_heads, 3, head_size) - unpack_qkv(info, query, key, value); - } - else - { - // Query: (batch_size, q_sequence_length, hidden_size) - std::vector q_dims{ - params.batch_size, params.q_sequence_length, num_heads, params.head_size}; - query = info.add_instruction(make_op("reshape", {{"dims", q_dims}}), query); - - key = args[1]; - - if(params.qkv_fomat == qkv_fomat_t::kv_packed) - { - // Packed KV: (batch_size, kv_sequence_length, num_heads, 2, head_size) - unpack_kv(info, key, value); - } - else - { - value = args[2]; - if(params.qkv_fomat == qkv_fomat_t::q_k_v) - { - // Key: (batch_size, kv_sequence_length, hidden_size) - // Value: (batch_size, kv_sequence_length, hidden_size_v) - std::vector k_dims{ - params.batch_size, params.kv_sequence_length, num_heads, params.head_size}; - std::vector v_dims{params.batch_size, - params.kv_sequence_length, - num_heads, - params.head_size_v}; - key = info.add_instruction(make_op("reshape", {{"dims", k_dims}}), key); - value = info.add_instruction(make_op("reshape", {{"dims", v_dims}}), value); - } - } - } - - // Target shape: (batch_size, num_heads, sequence_length, head_size) - std::vector perm{0, 2, 1, 3}; - query = info.add_instruction(make_op("transpose", {{"permutation", perm}}), query); - if(params.qkv_fomat != qkv_fomat_t::q_k_v_cross) - { - key = info.add_instruction(make_op("transpose", {{"permutation", perm}}), key); - value = info.add_instruction(make_op("transpose", {{"permutation", perm}}), value); - } - - float scale = 1 / std::sqrt(params.head_size); - if(contains(info.attributes, "scale")) - scale = parser.parse_value(info.attributes.at("scale")).at(); - - auto scale_literal = info.add_literal( - migraphx::literal{migraphx::shape{query->get_shape().type()}, {scale}}); - - auto key_transposed = - info.add_instruction(make_op("transpose", {{"permutation", {0, 1, 3, 2}}}), key); - - auto result = info.add_instruction(make_op("dot"), query, key_transposed); - result = info.add_common_op("mul", result, scale_literal); - result = info.add_instruction(make_op("softmax", {{"axis", -1}}), result); - result = info.add_instruction(make_op("dot"), result, value); - result = info.add_instruction(make_op("transpose", {{"permutation", perm}}), result); - result = info.add_instruction( - make_op( - "reshape", - {{"dims", {params.batch_size, params.q_sequence_length, params.hidden_size_v}}}), - result); - - return result; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_multinomial.cpp b/docker/rocm/migraphx/onnx/parse_multinomial.cpp deleted file mode 100644 index c890bd73a..000000000 --- a/docker/rocm/migraphx/onnx/parse_multinomial.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_multinomial : op_parser -{ - std::vector operators() const { return {{"Multinomial"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - if(args.empty()) - MIGRAPHX_THROW("PARSE_MULTINOMIAL: no arguments given"); - - int dtype = 6; - if(contains(info.attributes, "dtype")) - dtype = info.attributes.at("dtype").i(); - shape::type_t output_type = get_type(dtype); - - size_t sample_size = 1; - if(contains(info.attributes, "sample_size")) - sample_size = info.attributes.at("sample_size").i(); - else - MIGRAPHX_THROW("PARSE_MULTINOMIAL: sample_size not given"); - - // Use logarithmic math to scale probabilities while avoiding division by very - // small numbers. Scaling by the maximum makes very tiny ranges more - // tractable; any constant factor gives equivalent distr. since the Multinomial op. - // normalizes at runtime. - - // Subtract the per-batch maximum log-probability, making the per-batch max 0 - auto maxes = - info.add_instruction(migraphx::make_op("reduce_max", {{"axes", {1}}}), args[0]); - auto cdf = info.add_common_op("sub", args[0], maxes); - // Take the element-wise exponent to get probabilities in the range (0, 1] - cdf = info.add_instruction(migraphx::make_op("exp"), cdf); - // Compute the cumulative distribution function - cdf = info.add_instruction( - migraphx::make_op("prefix_scan_sum", {{"axis", 1}, {"exclusive", false}}), cdf); - - instruction_ref seed_input; - if(contains(info.attributes, "seed")) - { - float seed = info.attributes.at("seed").f(); - migraphx::shape s{migraphx::shape::float_type, {1}}; - std::vector data = {seed}; - seed_input = info.add_literal(migraphx::literal(s, data)); - } - else - { - seed_input = info.add_instruction(migraphx::make_op("random_seed")); - } - instruction_ref randoms; - - shape s0 = args[0]->get_shape(); - - if(s0.dynamic()) - { - // Dynamic batch_size will be taken from args[0]. The input argument to this should - // have a second dimension of sample_size. - std::vector dyn_dim_set; - dyn_dim_set.emplace_back(s0.dyn_dims().front()); - dyn_dim_set.emplace_back(shape::dynamic_dimension{sample_size, sample_size}); - - // read the input dimensions - auto dim_of = - info.add_instruction(migraphx::make_op("dimensions_of", {{"end", 2}}), args[0]); - - // The next two operations insert the value sample_size into the second array position - - // make an argument of (1, 0) - shape s(shape::int64_type, {2}); - std::vector data1{1, 0}; - auto l1 = info.add_literal(s, data1); - auto batch_arg = info.add_instruction(migraphx::make_op("mul"), dim_of, l1); - std::vector data2(2, 0); - // make an argument of (0, sample_size) - data2[1] = sample_size; - auto l2 = info.add_literal(s, data2); - auto alloc_shape = info.add_instruction(migraphx::make_op("add"), batch_arg, l2); - // alloc_shape should contain the input-based shape dimensions as its values at runtime, - // and its own shape is {2} - - // compile_shape is the shape used when compiling the Allocate op, and may be dynamic - migraphx::shape compile_shape = - migraphx::shape(s0.type(), {s0.dyn_dims().front(), {sample_size, sample_size}}); - - // Allocate on-device storage for the random values - auto alloc = info.add_instruction( - migraphx::make_op("allocate", {{"shape", to_value(compile_shape)}}), alloc_shape); - randoms = info.add_instruction(migraphx::make_op("random_uniform"), seed_input, alloc); - } - else - { - // use literal. The array populated by random_uniform may have any shape, as long its - // number of elements is batch_size * sample_size . - size_t batch_size = s0.lens().front(); - auto rand_dummy = info.add_literal(migraphx::literal{ - migraphx::shape{migraphx::shape::float_type, {batch_size, sample_size}}, - std::vector(batch_size * sample_size)}); - randoms = - info.add_instruction(migraphx::make_op("random_uniform"), seed_input, rand_dummy); - } - - return info.add_instruction( - migraphx::make_op("multinomial", {{"dtype", output_type}}), cdf, randoms); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_nonmaxsuppression.cpp b/docker/rocm/migraphx/onnx/parse_nonmaxsuppression.cpp deleted file mode 100644 index 212ee5123..000000000 --- a/docker/rocm/migraphx/onnx/parse_nonmaxsuppression.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_nonmaxsuppression : op_parser -{ - std::vector operators() const { return {{"NonMaxSuppression", "nonmaxsuppression"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - auto op = parser.load(opd.op_name, info); - op.from_value({{"use_dyn_output", parser.use_dyn_output}}); - return info.add_instruction(op, args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_nonzero.cpp b/docker/rocm/migraphx/onnx/parse_nonzero.cpp deleted file mode 100644 index 913972f7a..000000000 --- a/docker/rocm/migraphx/onnx/parse_nonzero.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -template -static std::vector nonzero_indices(const std::vector& data) -{ - std::vector indices; - for(std::size_t i = 0; i < data.size(); ++i) - { - if(not float_equal(data[i], 0)) - indices.push_back(i); - } - - return indices; -} - -struct parse_nonzero : op_parser -{ - std::vector operators() const { return {{"NonZero"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - migraphx::argument data_arg = args.back()->eval(); - if(data_arg.empty()) - { - return info.add_instruction(make_op("nonzero"), args); - } - else - { - std::vector indices; - data_arg.visit([&](auto val) { - using val_type = std::remove_cv_t; - std::vector vec_data; - vec_data.assign(val.begin(), val.end()); - indices = nonzero_indices(vec_data); - }); - - shape in_s = args[0]->get_shape(); - shape out_s{shape::int64_type, {in_s.lens().size(), indices.size()}}; - - std::vector out_data(out_s.elements()); - for(std::size_t i = 0; i < indices.size(); ++i) - { - auto idx = in_s.multi(indices[i]); - for(std::size_t j = 0; j < in_s.lens().size(); ++j) - { - out_data[out_s.index({j, i})] = idx[j]; - } - } - - return info.add_literal(literal(out_s, out_data)); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_onehot.cpp b/docker/rocm/migraphx/onnx/parse_onehot.cpp deleted file mode 100644 index 7cb307b15..000000000 --- a/docker/rocm/migraphx/onnx/parse_onehot.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_onehot : op_parser -{ - std::vector operators() const { return {{"OneHot"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - int64_t axis = -1; - if(contains(info.attributes, "axis")) - { - axis = info.attributes.at("axis").i(); - } - - migraphx::argument depth_arg = args[1]->eval(); - if(depth_arg.empty()) - { - return info.add_instruction(make_op("onehot", {{"axis", axis}}), args); - } - else - { - size_t depth_val = depth_arg.at(); - return info.add_instruction(make_op("onehot", {{"axis", axis}, {"depth", depth_val}}), - {args[0], args[2]}); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_pad.cpp b/docker/rocm/migraphx/onnx/parse_pad.cpp deleted file mode 100644 index c4be776b8..000000000 --- a/docker/rocm/migraphx/onnx/parse_pad.cpp +++ /dev/null @@ -1,273 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void calc_reflect_indices(std::vector& indices, const int64_t num_dims) -{ - int k = 0; - bool reversed = false; - // in reflect padding, if the num_pads > num_dims, - // compute the extra pad indices periodically, ex. ( 1, 2, 3, 2, 1, 0) - for(int& idx : indices) - { - if(k == num_dims - 1) - reversed = true; - if(k == 0) - reversed = false; - if(reversed) - k--; - else - k++; - idx = k; - } -} - -instruction_ref reflect_pad(const onnx_parser::node_info& info, - const std::vector& pads, - instruction_ref input) -{ - size_t num_dims = pads.size() / 2; - std::vector ldims(pads.begin(), pads.begin() + num_dims); - std::vector rdims(pads.begin() + num_dims, pads.end()); - assert(ldims.size() == rdims.size()); - - std::vector axes(num_dims); - std::iota(axes.begin(), axes.end(), int64_t{0}); - - // iterate over dimensions, starting from lowest dimension - for(int64_t i = num_dims - 1; i >= 0; i--) - { - auto axis = i; - auto lcount = ldims.at(i); - auto rcount = rdims.at(i); - if(lcount == 0 and rcount == 0) // no padding for current dim - continue; - - // calculate starts and ends for each iteration since shape may change - std::vector dims = input->get_shape().lens(); - std::vector starts(axes.size(), 0); - std::vector ends(dims.begin(), dims.end()); - std::vector slices; - - auto starts_it = starts.begin() + i; - auto ends_it = ends.begin() + i; - auto dims_it = dims.begin() + i; - - std::vector l_indices(lcount); - std::vector r_indices(rcount); - - // compute slice indices in a periodic fashion - calc_reflect_indices(l_indices, *dims_it); - calc_reflect_indices(r_indices, *dims_it); - - for(int idx : l_indices) - { - *starts_it = idx; - *ends_it = *starts_it + 1; - slices.push_back(info.add_instruction( - make_op("slice", {{"axes", axes}, {"starts", starts}, {"ends", ends}}), input)); - } - // when padding on the left side, the outermost pad should be at the beginning - std::reverse(slices.begin(), slices.end()); - slices.push_back(input); - for(int idx : r_indices) - { - *starts_it = *dims_it - idx - 1; - *ends_it = *starts_it + 1; - slices.push_back(info.add_instruction( - make_op("slice", {{"axes", axes}, {"starts", starts}, {"ends", ends}}), input)); - } - input = info.add_instruction(make_op("concat", {{"axis", axis}}), slices); - } - return input; -} - -struct parse_pad : op_parser -{ - std::vector operators() const { return {{"Pad"}}; } - - std::string parse_mode(const onnx_parser::node_info& info, - const std::vector& args) const - { - if(contains(info.attributes, "mode")) - { - auto mode = info.attributes.at("mode").s(); - if(mode == "reflect") - { - if(args.front()->get_shape().dynamic()) - { - MIGRAPHX_THROW("PARSE_PAD: reflect padding with dynamic shape not supported"); - } - } - else if(mode != "constant") - { - MIGRAPHX_THROW( - "PARSE_PAD: migraphx currently only supports constant and reflect padding"); - } - return mode; - } - else - { - // default mode - return "constant"; - } - } - - std::vector parse_pads(const onnx_parser::node_info& info, - const std::vector& args) const - { - std::vector pads{}; - if(args.size() >= 2) - { - auto pad_arg = args.at(1)->eval(); - check_arg_empty(pad_arg, "PARSE_PAD: `pads` input must be constant"); - pad_arg.visit([&](auto v) { pads.assign(v.begin(), v.end()); }); - } - else if(contains(info.attributes, "pads")) - { - auto&& pad_vals = info.attributes.at("pads").ints(); - pads = std::vector(pad_vals.begin(), pad_vals.end()); - } - else - { - MIGRAPHX_THROW("PARSE_PAD: `pads` must be available"); - } - return pads; - } - - float parse_constant_value(const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - float value = 0.0f; - if(args.size() >= 3 and args.at(2)->get_shape().elements() >= 1) - { - auto val_ins = args.at(2); - if(not val_ins->can_eval()) - { - MIGRAPHX_THROW("PARSE_PAD: input `value` must be constant"); - } - auto val_arg = val_ins->eval(); - if(val_arg.get_shape().elements() != 1) - { - MIGRAPHX_THROW("PARSE_PAD: `value` should contain only one element"); - } - value = val_arg.at(); - } - else if(contains(info.attributes, "value")) - { - value = parser.parse_value(info.attributes.at("value")).at(); - } - return value; - } - - std::vector parse_axes(const std::vector& args, - bool is_constant_mode) const - { - std::vector axes{}; - // axes is 3rd or 4th, depending on constant mode - auto pos = is_constant_mode ? 4 : 3; - if(args.size() >= pos) - { - auto axes_arg = args.at(pos - 1)->eval(); - check_arg_empty(axes_arg, "PARSE_PAD: variable `axes` input not supported"); - axes_arg.visit([&](auto v) { axes.assign(v.begin(), v.end()); }); - } - return axes; - } - - std::vector calculate_pads_with_axes(const std::vector& pads, - const std::vector& axes, - size_t input_rank) const - { - size_t num_axes = axes.size(); - if(num_axes * 2 != pads.size()) - { - MIGRAPHX_THROW("PARSE_PAD: number of elements of pads should be equal to 2 * " - "number of elements of axes"); - } - - std::vector new_pads(input_rank * 2); - for(size_t idx{0}; idx < num_axes; ++idx) - { - // axis can be negative - int64_t axis = axes[idx] < 0 ? input_rank + axes[idx] : axes[idx]; - // pad format is x1_begin, x2_begin, ... , x3_end, x4_end - new_pads[axis] = pads[idx]; - new_pads[axis + input_rank] = pads[idx + num_axes]; - } - return new_pads; - } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - std::vector pads = parse_pads(info, args); - - // check if padding is actually being done (at least one value is nonzero) - if(std::all_of(pads.begin(), pads.end(), [](const int& i) { return i == 0; })) - { - return info.add_instruction(make_op("identity"), args.front()); - } - - std::string mode = parse_mode(info, args); - bool is_constant_mode = mode == "constant"; - float value = is_constant_mode ? parse_constant_value(parser, info, args) : 0.0f; - std::vector axes = parse_axes(args, is_constant_mode); - size_t input_rank = args.front()->get_shape().ndim(); - - if(not axes.empty()) - { - pads = calculate_pads_with_axes(pads, axes, input_rank); - } - - if(pads.size() != input_rank * 2) - { - MIGRAPHX_THROW("PARSE_PAD: number of elements of pads should be equal to 2 * " - "input rank"); - } - - if(mode == "reflect") - { - return reflect_pad(info, pads, args.front()); - } - - return info.add_instruction(migraphx::make_op("pad", {{"pads", pads}, {"value", value}}), - args.front()); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_pooling.cpp b/docker/rocm/migraphx/onnx/parse_pooling.cpp deleted file mode 100644 index 853b93ed6..000000000 --- a/docker/rocm/migraphx/onnx/parse_pooling.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_pooling : op_parser -{ - std::vector operators() const - { - return { - {"AveragePool", "average"}, - {"GlobalAveragePool", "average"}, - {"GlobalMaxPool", "max"}, - {"MaxPool", "max"}, - {"LpPool", "lpnorm"}, - {"GlobalLpPool", "lpnorm"}, - }; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - return add_pooling_op(opd, std::move(info), args[0]); - }; -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_pow.cpp b/docker/rocm/migraphx/onnx/parse_pow.cpp deleted file mode 100644 index 23782de0a..000000000 --- a/docker/rocm/migraphx/onnx/parse_pow.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -auto compute_type(shape::type_t t1, shape::type_t t2) -{ - const static std::unordered_map op_order = {{shape::int8_type, 1}, - {shape::uint8_type, 2}, - {shape::int16_type, 3}, - {shape::uint16_type, 4}, - {shape::int32_type, 5}, - {shape::uint32_type, 6}, - {shape::int64_type, 7}, - {shape::uint64_type, 8}, - {shape::half_type, 9}, - {shape::float_type, 10}, - {shape::double_type, 11}}; - - int it1 = t1; - int it2 = t2; - if(not contains(op_order, it1) or not contains(op_order, it2)) - { - MIGRAPHX_THROW("PARSE_POW: Input data type not supported!"); - } - - return ((op_order.at(it1) >= op_order.at(it2)) ? t1 : t2); -} - -struct parse_pow : op_parser -{ - std::vector operators() const { return {{"Pow"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto type_base = args[0]->get_shape().type(); - auto type_exponent = args[1]->get_shape().type(); - - auto type_compute = compute_type(type_base, type_exponent); - if(type_compute != type_base) - { - args[0] = - info.add_instruction(make_op("convert", {{"target_type", type_compute}}), args[0]); - } - - if(type_compute != type_exponent) - { - args[1] = - info.add_instruction(make_op("convert", {{"target_type", type_compute}}), args[1]); - } - - auto ret = info.add_broadcastable_binary_op("pow", args[0], args[1]); - if(type_compute != type_base) - { - ret = info.add_instruction(make_op("convert", {{"target_type", type_base}}), ret); - } - - return ret; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_prefix_scan.cpp b/docker/rocm/migraphx/onnx/parse_prefix_scan.cpp deleted file mode 100644 index 2b0bf173b..000000000 --- a/docker/rocm/migraphx/onnx/parse_prefix_scan.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -instruction_ref parse_prefix_scan_oper(const std::string& op_name, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) -{ - migraphx::argument in = args[1]->eval(); - check_arg_empty(in, "PARSE_PREFIX_SCAN: axis - dynamic shape not supported"); - std::vector axis_in; - in.visit([&](auto input) { axis_in.assign(input.begin(), input.end()); }); - int64_t axis = axis_in[0]; - - bool exclusive = false; - bool reverse = false; - - if(contains(info.attributes, "exclusive")) - { - exclusive = parser.parse_value(info.attributes.at("exclusive")).at(); - } - - if(contains(info.attributes, "reverse")) - { - reverse = parser.parse_value(info.attributes.at("reverse")).at(); - } - - return info.add_instruction( - make_op(op_name, {{"axis", axis}, {"exclusive", exclusive}, {"reverse", reverse}}), - args[0]); -} - -struct parse_prefix_scan_op : op_parser -{ - std::vector operators() const { return {{"CumSum", "prefix_scan_sum"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - return parse_prefix_scan_oper(opd.op_name, parser, std::move(info), std::move(args)); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_qlinearbinary.cpp b/docker/rocm/migraphx/onnx/parse_qlinearbinary.cpp deleted file mode 100644 index 0e4468f4f..000000000 --- a/docker/rocm/migraphx/onnx/parse_qlinearbinary.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - ********************************************************************************* - * Reference: see QLinearAdd, QLinearMul in * - * https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md * - ********************************************************************************* - - com.microsoft.QLinearAdd - Performs element-wise binary addition on 8 bit data types (with Numpy-style broadcasting support). - - C = (A_scale * (A - A_zero_point) + B_scale * (B - B_zero_point))/C_scale + C_zero_point - - Version - This version of the operator has been available since version 1 of the 'com.microsoft' operator - set. - - com.microsoft.QLinearMul - Performs element-wise binary multiplication on 8 bit data types (with Numpy-style broadcasting - support). - - C = ((A - A_zero_point) * (B - B_zero_point)) * (A_scale * B_scale)/C_scale + C_zero_point - - Version - This version of the operator has been available since version 1 of the 'com.microsoft' operator - set. - - General definition of binary QLinear* ops: - Inputs (7 - 8) - A : T - First operand. - - A_scale : tensor(float) - Input A's scale. It's a scalar, which means a per-tensor/layer quantization. - - A_zero_point (optional) : T - Input A zero point. Default value is 0 if it's not specified. It's a scalar, which means a - per-tensor/layer quantization. - - B : T - Second operand. - - B_scale : tensor(float) - Input B's scale. It's a scalar, which means a per-tensor/layer quantization. - - B_zero_point (optional) : T - Input B zero point. Default value is 0 if it's not specified. It's a scalar, which means a - per-tensor/layer quantization. - - C_scale : tensor(float) - Output scale. It's a scalar, which means a per-tensor/layer quantization. - - C_zero_point (optional) : T - - Output zero point. Default value is 0 if it's not specified. It's a scalar, which means a - per-tensor/layer quantization. - - Outputs - C : T - Result, has same element type as two inputs - - Type Constraints - T : tensor(uint8), tensor(int8) - Constrain input and output types to 8 bit signed and unsigned tensors. - -*/ - -struct parse_qlinearbinary : op_parser -{ - std::vector operators() const - { - return {{"QLinearAdd", "add"}, {"QLinearMul", "mul"}}; - } - - // basic type checking for binary QLinear Operator - void check_inputs(const std::vector& args, const std::string& onnx_name) const - { - if(args.size() < 7) - MIGRAPHX_THROW(onnx_name + ": missing inputs"); - - const auto& in_a = args[0]; - const auto& in_b = args[3]; - - auto sh_a = in_a->get_shape(); - auto sh_b = in_b->get_shape(); - - auto type_a = sh_a.type(); - auto type_b = sh_b.type(); - if(type_a != migraphx::shape::int8_type and type_a != migraphx::shape::uint8_type) - MIGRAPHX_THROW(onnx_name + ": unsupported input type"); - if(type_b != migraphx::shape::int8_type and type_b != migraphx::shape::uint8_type) - MIGRAPHX_THROW(onnx_name + ": unsupported input type"); - if(type_a != type_b) - MIGRAPHX_THROW(onnx_name + ": mismatched input types"); - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - const std::vector& args) const - { - check_inputs(args, opd.onnx_name); - - // A - const auto& in_a = args[0]; - const auto& in_scale_a = args[1]; - const auto& in_zero_pt_a = args[2]; - - auto dquant_a = bcast_qdq_instr("dequantizelinear", in_a, in_scale_a, in_zero_pt_a, info); - - // B - const auto& in_b = args[3]; - const auto& in_scale_b = args[4]; - const auto& in_zero_pt_b = args[5]; - auto dquant_b = bcast_qdq_instr("dequantizelinear", in_b, in_scale_b, in_zero_pt_b, info); - - // C = op(A, B) - auto out_c = info.add_common_op(opd.op_name, dquant_a, dquant_b); - - const auto& in_scale_c = args[6]; - - // zero_pt for C is supplied as the last optional argument.. - if(args.size() == 8) - return (bcast_qdq_instr("quantizelinear", out_c, in_scale_c, args[7], info)); - - // if no zero_pt: just broadcast the scale.. - auto bcast_scale_c = bcast_scalar_instr(out_c->get_shape(), in_scale_c, info); - return (info.add_instruction(migraphx::make_op("quantizelinear"), out_c, bcast_scale_c)); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_qlinearconcat.cpp b/docker/rocm/migraphx/onnx/parse_qlinearconcat.cpp deleted file mode 100644 index 3cee05c30..000000000 --- a/docker/rocm/migraphx/onnx/parse_qlinearconcat.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_qlinearconcat : op_parser -{ - std::vector operators() const { return {{"QLinearConcat"}}; } - - // basic type checking for QLinearConcat Operator - void check_inputs(const std::vector& args) const - { - auto args_size = args.size(); - // at least 5 input tensors: - // 1. is Y_scale: tensor(float) - // 2. is Y_zero_pont: tensor(uint8)/tensor(int8) - // remaining is a sequence of : - // 3. Tensor: tensor(uint8)/tensor(int8) - // 4. Scale: tensor(float), - // 5. ZeroPoint: tensor(uint8)/tensor(int8) tensors - // Size can be 5, 8, 11 ... - if((args_size < 5) or ((args_size - 2) % 3 != 0)) - MIGRAPHX_THROW("QLINEARCONCAT: missing inputs"); - - auto y_zp = args[1]; - auto y_zp_type = y_zp->get_shape().type(); - if(y_zp_type != migraphx::shape::int8_type and y_zp_type != migraphx::shape::uint8_type) - MIGRAPHX_THROW("QLINEARCONCAT: unsupported output type"); - - auto t0_type = args[2]->get_shape().type(); - if(t0_type != migraphx::shape::int8_type and t0_type != migraphx::shape::uint8_type) - MIGRAPHX_THROW("QLINEARCONCAT: unsupported input type"); - for(auto idx = 2; idx < args.size(); idx += 3) - { - if((args[idx]->get_shape().type() != t0_type) or - (args[idx + 2]->get_shape().type() != t0_type)) - { - MIGRAPHX_THROW("QLINEARCONCAT: mismatching input types"); - } - } - } - - instruction_ref parse(const op_desc& /* opd */, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - check_inputs(args); - if(not contains(info.attributes, "axis")) - MIGRAPHX_THROW("QLINEARCONCAT: missing axis attribute"); - - auto axis = parser.parse_value(info.attributes.at("axis")).template at(); - std::vector tmp; - for(auto idx = 2; idx < args.size(); idx += 3) - { - auto data_tensor = args[idx]; - auto scale = args[idx + 1]; - auto zero_pt = args[idx + 2]; - tmp.push_back(bcast_qdq_instr("dequantizelinear", data_tensor, scale, zero_pt, info)); - } - auto y = info.add_instruction(migraphx::make_op("concat", {{"axis", axis}}), tmp); - - auto y_scale = args[0]; - auto y_zero_pt = args[1]; - - return bcast_qdq_instr("quantizelinear", y, y_scale, y_zero_pt, info); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_qlinearconv.cpp b/docker/rocm/migraphx/onnx/parse_qlinearconv.cpp deleted file mode 100644 index 26f2f7b91..000000000 --- a/docker/rocm/migraphx/onnx/parse_qlinearconv.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - ********************************************************************************* - * Reference: see QLinearConv in * - * https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md * - ********************************************************************************* - -com.microsoft.QLinearConv - -Version -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -ATTRIBUTES: -auto_pad : string -channels_last : int -dilations : list of ints -group : int -kernel_shape : list of ints -pads : list of ints -strides : list of ints - -INPUTS (8 - 9): -x : T1 -x_scale : tensor(float) -x_zero_point : T1 -w : T2 -w_scale : tensor(float) -w_zero_point : T2 -y_scale : tensor(float) -y_zero_point : T3 -B (optional) : T4 - -OUTPUTS: -y : T3 - -Type Constraints: -T1 : tensor(int8), tensor(uint8) -T2 : tensor(int8), tensor(uint8) -T3 : tensor(int8), tensor(uint8) -T4 : tensor(int32) - -More details also at: -https://xadupre.github.io/draft/onnx/onnx_doc_folder/onnx__QLinearConv.html - -*/ - -struct parse_qlinearconv : op_parser -{ - std::vector operators() const { return {{"QLinearConv"}}; } - - // basic type checking for QLinearConv Operator - void check_inputs(const std::vector& inp_arg) const - { - if(inp_arg.size() < 8) - MIGRAPHX_THROW("QLINEARCONV: missing inputs"); - - const instruction_ref& in_x = inp_arg[0]; - const instruction_ref& in_scale_x = inp_arg[1]; - const instruction_ref& in_w = inp_arg[3]; - const instruction_ref& in_scale_w = inp_arg[4]; - const instruction_ref& in_scale_y = inp_arg[6]; - - auto sh_x = in_x->get_shape(); - auto sh_w = in_w->get_shape(); - auto type_x = sh_x.type(); - auto type_w = sh_w.type(); - - assert(in_x->get_shape().ndim() > 2); - - if(type_x != shape::int8_type and type_x != shape::uint8_type) - MIGRAPHX_THROW("QLINEARCONV: unsupported input type"); - if(type_w != shape::int8_type and type_w != shape::uint8_type) - MIGRAPHX_THROW("QLINEARCONV: unsupported weight type"); - if(in_scale_x->get_shape().type() != shape::float_type) - MIGRAPHX_THROW("QLINEARCONV x scale type should be float"); - if(in_scale_w->get_shape().type() != shape::float_type) - MIGRAPHX_THROW("QLINEARCONV: wt scale type should be float"); - if(in_scale_y->get_shape().type() != shape::float_type) - MIGRAPHX_THROW("QLINEARCONV: y scale type should be float"); - if(inp_arg.size() > 8 and inp_arg[8]->get_shape().type() != shape::int32_type) - MIGRAPHX_THROW("QLINEARCONV y bias should be int32"); - } - - // process all attributes of QLinearConv Operator.. - value process_attributes(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - value values; - - const auto& in_x = args[0]; - const auto& wt = args[3]; - - size_t kdims = in_x->get_shape().ndim() - 2; - - check_padding_mode(info, opd.onnx_name); - - values["stride"] = std::vector(kdims, 1); - values["dilation"] = std::vector(kdims, 1); - values["padding"] = std::vector(kdims, 0); - values["group"] = 1; - - if(contains(info.attributes, "group")) - values["group"] = parser.parse_value(info.attributes.at("group")).template at(); - - if(contains(info.attributes, "strides")) - { - std::vector st; - copy(info.attributes.at("strides").ints(), std::back_inserter(st)); - check_attr_sizes(kdims, st.size(), "QLINEARCONV: inconsistent strides"); - values["stride"] = st; - } - - if(contains(info.attributes, "dilations")) - { - std::vector dil; - copy(info.attributes.at("dilations").ints(), std::back_inserter(dil)); - check_attr_sizes(kdims, dil.size(), "QLINEARCONV: inconsistent dilations"); - values["dilation"] = dil; - } - - if(contains(info.attributes, "pads")) - { - std::vector pads; - copy(info.attributes.at("pads").ints(), std::back_inserter(pads)); - check_attr_sizes(kdims, pads.size() / 2, "QLINEARCONV: inconsistent padding"); - values["padding"] = pads; - } - else if(contains(info.attributes, "auto_pad")) - { - auto in_lens = in_x->get_shape().lens(); - auto wt_lens = wt->get_shape().lens(); - std::vector k_lens(wt_lens.begin() + 2, wt_lens.end()); - std::vector pads = values["padding"].to_vector(); - cal_auto_padding_size( - info, values, k_lens, values["dilation"].to_vector(), in_lens, pads); - values["padding"] = pads; - } - - recalc_conv_attributes(values, kdims); - - return values; - } - - instruction_ref add_bias_to_conv(const instruction_ref bias_arg, - const instruction_ref conv_instr, - const onnx_parser::node_info& info) const - { - auto conv_sh = conv_instr->get_shape(); - auto conv_lens = conv_sh.lens(); - auto conv_type = conv_sh.type(); - - auto broadcast_bias = info.add_instruction( - migraphx::make_op("broadcast", {{"axis", 1}, {"out_lens", conv_lens}}), bias_arg); - auto f_bias = - info.add_instruction(make_op("convert", {{"target_type", conv_type}}), broadcast_bias); - - return info.add_instruction(migraphx::make_op("add"), conv_instr, f_bias); - }; - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - check_inputs(args); - - auto values = process_attributes(opd, parser, info, args); - - // input: quantized x, scale, zero_pt - const instruction_ref& in_x = args[0]; - const instruction_ref& in_scale_x = args[1]; - const instruction_ref& in_zero_pt_x = args[2]; - - // input: quantized weights, scale, zero_pt - const instruction_ref& in_w = args[3]; - const instruction_ref& in_scale_w = args[4]; - const instruction_ref& in_zero_pt_w = args[5]; - - // for the dequantized output y: scale & zero_pt - const instruction_ref& in_scale_y = args[6]; - const instruction_ref& in_zero_pt_y = args[7]; - - auto dquant_x = bcast_qdq_instr("dequantizelinear", in_x, in_scale_x, in_zero_pt_x, info); - - auto dquant_w = bcast_qdq_instr("dequantizelinear", in_w, in_scale_w, in_zero_pt_w, info); - - auto conv_op = migraphx::make_op("convolution", values); - - auto conv_x_w = info.add_instruction(conv_op, dquant_x, dquant_w); - - // Biases, if any.. : is an optional argument. - if(args.size() > 8) - conv_x_w = add_bias_to_conv(args[8], conv_x_w, info); - - return bcast_qdq_instr("quantizelinear", conv_x_w, in_scale_y, in_zero_pt_y, info); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_qlinearmatmul.cpp b/docker/rocm/migraphx/onnx/parse_qlinearmatmul.cpp deleted file mode 100644 index 1b430ab6f..000000000 --- a/docker/rocm/migraphx/onnx/parse_qlinearmatmul.cpp +++ /dev/null @@ -1,198 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - ********************************************************************************* - * Reference: see QLinearMatMul in * - * https://onnx.ai/onnx/operators/onnx__QLinearMatMul.html * - ********************************************************************************* - -Matrix product that behaves like numpy.matmul: - -https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.matmul.html. It consumes two -quantized input tensors, their scales and zero points, scale and zero point of output, and computes -the quantized output. The quantization formula is y = saturate((x / y_scale) + y_zero_point). For (x -/ y_scale), it is rounding to nearest ties to even. Refer to https://en.wikipedia.org/wiki/Rounding -for details. Scale and zero point must have same shape. They must be either scalar (per tensor) or -N-D tensor (per row for ‘a’ and per column for ‘b’). Scalar refers to per tensor quantization -whereas N-D refers to per row or per column quantization. If the input is 2D of shape [M, K] then -zero point and scale tensor may be an M element vector [v_1, v_2, …, v_M] for per row quantization -and K element vector of shape [v_1, v_2, …, v_K] for per column quantization. If the input is N-D -tensor with shape [D1, D2, M, K] then zero point and scale tensor may have shape [D1, D2, M, 1] for -per row quantization and shape [D1, D2, 1, K] for per column quantization. Production must never -overflow, and accumulation may overflow if and only if in 32 bits. - -Inputs -a (heterogeneous) - T1: N-dimensional quantized matrix a - -a_scale (heterogeneous) - tensor(float): scale of quantized input a - -a_zero_point (heterogeneous) - T1: zero point of quantized input a - -b (heterogeneous) - T2: N-dimensional quantized matrix b - -b_scale (heterogeneous) - tensor(float): scale of quantized input b - -b_zero_point (heterogeneous) - T2: zero point of quantized input b - -y_scale (heterogeneous) - tensor(float): scale of quantized output y - -y_zero_point (heterogeneous) - T3: zero point of quantized output y - -Outputs -y (heterogeneous) - T3: Quantized matrix multiply results from a * b - -Type Constraints -T1 in ( tensor(int8), tensor(uint8) ): Constrain input a and its zero point data type to 8-bit -integer tensor. - -T2 in ( tensor(int8), tensor(uint8) ): Constrain input b and its zero point data type to 8-bit -integer tensor. - -T3 in ( tensor(int8), tensor(uint8) ): Constrain output y and its zero point data type to 8-bit -integer tensor. - -*/ - -struct parse_qlinearmatmul : op_parser -{ - std::vector operators() const { return {{"QLinearMatMul"}}; } - - // basic type checking for QLinearMatMul Operator - - void check_inputs(const std::vector& args) const - { - if(args.size() < 8) - MIGRAPHX_THROW("QLINEARMATMUL: missing inputs"); - - const auto& in_a = args[0]; - const auto& in_b = args[3]; - - auto sh_a = in_a->get_shape(); - auto sh_b = in_b->get_shape(); - - auto type_a = sh_a.type(); - auto type_b = sh_b.type(); - if(type_a != migraphx::shape::int8_type and type_a != migraphx::shape::uint8_type) - MIGRAPHX_THROW("QLINEARMATMUL: unsupported input type"); - if(type_b != migraphx::shape::int8_type and type_b != migraphx::shape::uint8_type) - MIGRAPHX_THROW("QLINEARMATMUL: unsupported input type"); - - auto lens_a = sh_a.lens(); - auto lens_b = sh_b.lens(); - - size_t dim_a = lens_a.size(); - size_t dim_b = lens_b.size(); - - if(dim_a == 0 or dim_b == 0) - MIGRAPHX_THROW("QLINEARMATMUL: empty input"); - - // broadcast supported if either is 1-D -- the other can be a 2-D tensor. - // if it is 1-D, just prepend/append that lens and check further constraints.. - if(dim_a == 1) - { - lens_a.insert(lens_a.begin(), 1); - dim_a++; - } - if(dim_b == 1) - { - lens_b.push_back(1); - dim_b++; - } - - // 2-D or higher-order mat mul - if(dim_a != dim_b or *lens_a.rbegin() != *(lens_b.rbegin() + 1) or - not std::equal(lens_a.rbegin() + 2, lens_a.rend(), lens_b.rbegin() + 2, lens_b.rend())) - MIGRAPHX_THROW("QLINEARMATMUL: mismatched input dimensions"); - - if(migraphx::any_of({args[1], args[2], args[4], args[5]}, - [](auto arg) { return not arg->get_shape().scalar(); })) - MIGRAPHX_THROW("QLINEARMATMUL: unsupported row/column quantization"); - } - - instruction_ref parse(const op_desc& /* opd */, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - const std::vector& args) const - { - check_inputs(args); - - // A - const auto& in_a = args[0]; - const auto& in_scale_a = args[1]; - const auto& in_zero_pt_a = args[2]; - auto dquant_a = bcast_qdq_instr("dequantizelinear", in_a, in_scale_a, in_zero_pt_a, info); - - // B - const auto& in_b = args[3]; - const auto& in_scale_b = args[4]; - const auto& in_zero_pt_b = args[5]; - auto dquant_b = bcast_qdq_instr("dequantizelinear", in_b, in_scale_b, in_zero_pt_b, info); - - bool is_a_prepended = false; - bool is_b_appended = false; - - // un-squeeze either tensor if 1-D. - if(in_a->get_shape().ndim() == 1) - { - is_a_prepended = true; - dquant_a = info.add_instruction(make_op("unsqueeze", {{"axes", {0}}}), dquant_a); - } - if(in_b->get_shape().ndim() == 1) - { - is_b_appended = true; - dquant_b = info.add_instruction(make_op("unsqueeze", {{"axes", {1}}}), dquant_b); - } - - // Y = A * B - auto out_y = info.add_instruction(migraphx::make_op("dot"), dquant_a, dquant_b); - - // squeeze just once if necessary.. not twice. - if(is_a_prepended) - out_y = info.add_instruction(make_op("squeeze", {{"axes", {0}}}), out_y); - else if(is_b_appended) - out_y = info.add_instruction(make_op("squeeze", {{"axes", {1}}}), out_y); - - const auto& scale_y = args[6]; - const auto& zero_pt_y = args[7]; - - return bcast_qdq_instr("quantizelinear", out_y, scale_y, zero_pt_y, info); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_qlinearpooling.cpp b/docker/rocm/migraphx/onnx/parse_qlinearpooling.cpp deleted file mode 100644 index 245fc520d..000000000 --- a/docker/rocm/migraphx/onnx/parse_qlinearpooling.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - ********************************************************************************* - * Reference: see QLinearAveragePool and QLinearGlobalAveragePool in * - * github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md * - ********************************************************************************* - */ - -struct parse_qlinearpooling : op_parser -{ - std::vector operators() const - { - return {{"QLinearGlobalAveragePool", "average"}, {"QLinearAveragePool", "average"}}; - } - - void check_inputs(const op_desc& opd, const std::vector& args) const - { - const auto& in_x = args[0]; - const auto onnx_name = opd.onnx_name; - - if(in_x->get_shape().ndim() <= 2) - MIGRAPHX_THROW(onnx_name + ": input dimensions too small"); - - auto type_x = in_x->get_shape().type(); - if(type_x != migraphx::shape::int8_type and type_x != migraphx::shape::uint8_type) - MIGRAPHX_THROW(onnx_name + ": unsupported input type"); - - const auto& zero_pt_x = args[2]; - if(type_x != zero_pt_x->get_shape().type()) - MIGRAPHX_THROW(onnx_name + ": mismatched type: input zero point"); - - if(args.size() == 5) - { - const auto& zero_pt_y = args[4]; - if(type_x != zero_pt_y->get_shape().type()) - MIGRAPHX_THROW(onnx_name + ": mismatched type: output zero point"); - } - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - if(contains(info.attributes, "channel_last")) - { - int channels_last = - parser.parse_value(info.attributes.at("channels_last")).template at(); - if(channels_last != 0) - MIGRAPHX_THROW(opd.onnx_name + ": channels_last (N x D1..Dn x C) is not supported"); - } - - check_inputs(opd, args); - - // Input: X - - const auto& in_x = args[0]; - const auto& scale_x = args[1]; - const auto& zero_pt_x = args[2]; - auto dquant_x = bcast_qdq_instr("dequantizelinear", in_x, scale_x, zero_pt_x, info); - - // Output Y = pooling_op(X) - - auto out_y = add_pooling_op(opd, info, dquant_x); - - const auto& in_scale_y = args[3]; - // zero_pt for Y is supplied as the last optional argument.. - if(args.size() == 5) - return (bcast_qdq_instr("quantizelinear", out_y, in_scale_y, args[4], info)); - - // if no zero_pt: just broadcast the scale.. - auto bcast_scale_y = bcast_scalar_instr(out_y->get_shape(), in_scale_y, info); - return (info.add_instruction(migraphx::make_op("quantizelinear"), out_y, bcast_scale_y)); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_qlinearunary.cpp b/docker/rocm/migraphx/onnx/parse_qlinearunary.cpp deleted file mode 100644 index 6287ece5b..000000000 --- a/docker/rocm/migraphx/onnx/parse_qlinearunary.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - ********************************************************************************* - * Reference: see QLinearSigmoid, QLinearLeakyRelu in * - * https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md * - ********************************************************************************* - -com.microsoft.QLinearSigmoid -QLinearSigmoid takes quantized input data (Tensor), and quantize parameter for output, and produces -one output data (Tensor) where the function f(x) = quantize(Sigmoid(dequantize(x))), is applied to -the data tensor elementwise. Where the function Sigmoid(x) = 1 / (1 + exp(-x)) - -Version -This version of the operator has been available since version 1 of the 'com.microsoft' operator -set. - -***************************************************************************************************** - -com.microsoft.QLinearLeakyRelu -QLinearLeakyRelu takes quantized input data (Tensor), an argument alpha, and quantize parameter for -output, and produces one output data (Tensor) where the function f(x) = quantize(alpha * -dequantize(x)) for dequantize(x) < 0, f(x) = quantize(dequantize(x)) for dequantize(x) >= 0, is -applied to the data tensor elementwise. - -Version -This version of the operator has been available since version 1 of the 'com.microsoft' operator set. - -Attributes -alpha : float -Coefficient of leakage. - -****************************************************************************************************** - -Generic input layout of QLinear unary operators: - -Inputs (4 - 5) -X : T -Input tensor - -X_scale : tensor(float) -Input X's scale. It's a scalar, which means a per-tensor/layer quantization. - -X_zero_point (optional) : T -Input X's zero point. Default value is 0 if it's not specified. It's a scalar, which means a -per-tensor/layer quantization. - -Y_scale : tensor(float) Output Y's scale. It's a scalar, which means -a per-tensor/layer quantization. - -Y_zero_point (optional) : T Output Y's zero point. Default value is -0 if it's not specified. It's a scalar, which means a per-tensor/layer quantization. - -Outputs -Y : T -Output tensor - -Type Constraints -T : tensor(uint8), tensor(int8) -Constrain input and output types to 8 bit tensors. - -*/ - -struct parse_qlinearunary : op_parser -{ - std::vector operators() const - { - return {{"QLinearSigmoid", "sigmoid"}, {"QLinearLeakyRelu", "leaky_relu"}}; - } - - void check_inputs(const op_desc& opd, const std::vector& args) const - { - if(args.size() < 4) - MIGRAPHX_THROW(opd.op_name + ": missing inputs"); - - const auto& in_x = args[0]; - - auto sh_x = in_x->get_shape(); - auto type_x = sh_x.type(); - if(type_x != migraphx::shape::int8_type and type_x != migraphx::shape::uint8_type) - MIGRAPHX_THROW(opd.op_name + ": unsupported input type"); - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - check_inputs(opd, args); - - // X - const auto& in_x = args[0]; - const auto& in_scale_x = args[1]; - const auto& in_zero_pt_x = args[2]; - - auto dquant_x = bcast_qdq_instr("dequantizelinear", in_x, in_scale_x, in_zero_pt_x, info); - - // Y = (op(dequantizelinear(x)) - auto op = parser.load(opd.op_name, info); - auto y = info.add_instruction(op, dquant_x); - - const auto& in_scale_y = args[3]; - - // zero_pt for Y is supplied as the last optional argument.. - if(args.size() == 5) - return (bcast_qdq_instr("quantizelinear", y, in_scale_y, args[4], info)); - - // if no zero_pt: just broadcast the scale.. - auto bcast_scale_sigm = bcast_scalar_instr(y->get_shape(), in_scale_y, info); - return (info.add_instruction(migraphx::make_op("quantizelinear"), y, bcast_scale_sigm)); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_quantizelinear.cpp b/docker/rocm/migraphx/onnx/parse_quantizelinear.cpp deleted file mode 100644 index 92a773ae6..000000000 --- a/docker/rocm/migraphx/onnx/parse_quantizelinear.cpp +++ /dev/null @@ -1,115 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_quantizelinear : op_parser -{ - std::vector operators() const { return {{"QuantizeLinear"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - if(args.size() < 2 or args.size() > 3) - { - MIGRAPHX_THROW("QuantizeLinear: must have either 2 or 3 inputs, " + - std::to_string(args.size()) + " input(s) provided"); - } - - // Starting with version 19 ONNX introduced the constraint that x and y_scale types must be - // the same - if(parser.opset_version >= 19 and - args[0]->get_shape().type() != args[1]->get_shape().type()) - { - MIGRAPHX_THROW("QuantizeLinear: x and y_scale must be of same type"); - } - - if(args.size() == 3 and args[1]->get_shape().lens() != args[2]->get_shape().lens()) - { - MIGRAPHX_THROW( - "QuantizeLinear: y_scale and y_zero_point shapes must be equal. Provided y_scale " - "shape: " + - to_string_range(args[1]->get_shape().lens()) + - ", provided y_zero_point shape: " + to_string_range(args[2]->get_shape().lens())); - } - - int axis = 1; - if(contains(info.attributes, "axis")) - axis = info.attributes.at("axis").i(); - - int block_size = 0; - if(contains(info.attributes, "block_size")) - block_size = info.attributes.at("block_size").i(); - - std::optional output_type; - if(contains(info.attributes, "output_dtype")) - { - output_type = get_type(info.attributes.at("output_dtype").i()); - } - - if(output_type.has_value() and args.size() == 3 and - *output_type != args[2]->get_shape().type()) - { - MIGRAPHX_THROW( - "QuantizeLinear: output_type and y_zero_point type must match. output_type: " + - to_string(*output_type) + - +", y_zero_point type: " + to_string(args[2]->get_shape().type())); - } - - args = transform_quantize_dequantize_linear_inputs( - info, opd.onnx_name, block_size, axis, args); - - if(parser.opset_version < 19) - { - auto common_type = common_shape({args[0]->get_shape(), args[1]->get_shape()}).type(); - std::transform(args.begin(), args.begin() + 2, args.begin(), [&](auto ins) { - if(ins->get_shape().type() != common_type) - ins = info.add_instruction(make_op("convert", {{"target_type", common_type}}), - ins); - return ins; - }); - } - - if(output_type.has_value()) - return info.add_instruction(make_op("quantizelinear", {{"out_type", *output_type}}), - args); - else - return info.add_instruction(make_op("quantizelinear"), args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_randomnormal_ops.cpp b/docker/rocm/migraphx/onnx/parse_randomnormal_ops.cpp deleted file mode 100644 index e7b540d61..000000000 --- a/docker/rocm/migraphx/onnx/parse_randomnormal_ops.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_randomnormal_ops : op_parser -{ - std::set valid_types = {shape::float_type, shape::half_type, shape::double_type}; - - std::vector operators() const { return {{"RandomNormal"}, {"RandomNormalLike"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - int dtype = 1; - bool use_dtype = false; - if(contains(info.attributes, "dtype")) - { - dtype = info.attributes.at("dtype").i(); - use_dtype = true; - } - shape::type_t out_type = get_type(dtype); - if(not contains(valid_types, out_type)) - MIGRAPHX_THROW(opd.onnx_name + ": invalid output type: " + std::to_string(dtype) + - ". Valid types are 1 (float), 10 (half), and 11 (double)."); - - float mean = 0.0; - if(contains(info.attributes, "mean")) - mean = info.attributes.at("mean").f(); - - float scale = 1.0; - if(contains(info.attributes, "scale")) - scale = info.attributes.at("scale").f(); - - shape out_shape; - if(contains(info.attributes, "shape")) - { - // RandomNormal: - // output type and shape must come from attributes - std::vector out_lens; - literal ls = parser.parse_value(info.attributes.at("shape")); - ls.visit([&](auto s) { out_lens.assign(s.begin(), s.end()); }); - out_shape = shape{out_type, out_lens}; - } - else if(args.size() == 1) - { - // RandomNormalLike: - // output type and shape are the same as the input's by default - // dtype is used instead when attribute is set - if(not contains(valid_types, args[0]->get_shape().type())) - MIGRAPHX_THROW(opd.onnx_name + ": invalid output type: " + - std::to_string(args[0]->get_shape().type()) + - ". Valid types are float, half, and double."); - out_shape = - use_dtype ? shape{out_type, args[0]->get_shape().lens()} : args[0]->get_shape(); - } - else - { - MIGRAPHX_THROW(opd.onnx_name + - ": cannot deduce shape without shape attribute or argument."); - } - - std::mt19937 gen(std::chrono::high_resolution_clock::now().time_since_epoch().count()); - if(contains(info.attributes, "seed")) - gen.seed(info.attributes.at("seed").f()); - - std::normal_distribution<> d(mean, scale); - std::vector rand_vals(out_shape.elements()); - std::generate(rand_vals.begin(), rand_vals.end(), [&]() { return d(gen); }); - - return info.add_literal(literal{out_shape, rand_vals}); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_randomuniform_ops.cpp b/docker/rocm/migraphx/onnx/parse_randomuniform_ops.cpp deleted file mode 100644 index 52988874b..000000000 --- a/docker/rocm/migraphx/onnx/parse_randomuniform_ops.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_randomuniform_ops : op_parser -{ - std::set valid_types = {shape::float_type, shape::half_type, shape::double_type}; - - std::vector operators() const { return {{"RandomUniform"}, {"RandomUniformLike"}}; } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - int dtype = 1; - bool use_dtype = false; - if(contains(info.attributes, "dtype")) - { - dtype = info.attributes.at("dtype").i(); - use_dtype = true; - } - shape::type_t out_type = get_type(dtype); - if(not contains(valid_types, out_type)) - MIGRAPHX_THROW(opd.onnx_name + ": invalid output type: " + std::to_string(dtype) + - ". Valid types are 1 (float), 10 (half), and 11 (double)."); - - float high = 1.0; - if(contains(info.attributes, "high")) - high = info.attributes.at("high").f(); - - float low = 0.0; - if(contains(info.attributes, "low")) - low = info.attributes.at("low").f(); - - shape out_shape; - if(contains(info.attributes, "shape")) - { - // RandomUniform: - // output type and shape must come from attributes - std::vector out_lens; - literal ls = parser.parse_value(info.attributes.at("shape")); - ls.visit([&](auto s) { out_lens.assign(s.begin(), s.end()); }); - out_shape = shape{out_type, out_lens}; - } - else if(args.size() == 1) - { - // RandomUniformLike: - // output type and shape are the same as the input by default - // dtype is used instead when attribute is set - if(not contains(valid_types, args[0]->get_shape().type())) - MIGRAPHX_THROW(opd.onnx_name + ": invalid output type: " + - std::to_string(args[0]->get_shape().type()) + - ". Valid types are float, half, and double."); - out_shape = - use_dtype ? shape{out_type, args[0]->get_shape().lens()} : args[0]->get_shape(); - } - else - { - MIGRAPHX_THROW(opd.onnx_name + - ": cannot deduce shape without shape attribute or argument."); - } - - std::mt19937 gen(std::chrono::high_resolution_clock::now().time_since_epoch().count()); - if(contains(info.attributes, "seed")) - gen.seed(info.attributes.at("seed").f()); - - std::uniform_real_distribution<> d(low, high); - std::vector rand_vals(out_shape.elements()); - std::generate(rand_vals.begin(), rand_vals.end(), [&]() { return d(gen); }); - - return info.add_literal(literal{out_shape, rand_vals}); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_range.cpp b/docker/rocm/migraphx/onnx/parse_range.cpp deleted file mode 100644 index 2bf346718..000000000 --- a/docker/rocm/migraphx/onnx/parse_range.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_range : op_parser -{ - std::vector operators() const { return {{"Range"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - auto start_arg = args[0]->eval(); - check_arg_empty(start_arg, "PARSE_RANGE: start arg dynamic shape is not supported"); - auto limit_arg = args[1]->eval(); - check_arg_empty(limit_arg, "PARSE_RANGE: limit arg dynamic shape is not supported"); - auto delta_arg = args[2]->eval(); - check_arg_empty(delta_arg, "PARSE_RANGE: delta arg dynamic shape is not supported"); - - assert(args[0]->get_shape().elements() == 1 and args[1]->get_shape().elements() == 1 and - args[2]->get_shape().elements() == 1); - - instruction_ref l0; - - visit_all(start_arg, limit_arg, delta_arg)([&](auto start, auto limit, auto delta) { - auto start_val = start.front(); - auto limit_val = limit.front(); - auto delta_val = delta.front(); - - size_t num_elements = - ceil(static_cast(limit_val - start_val) / static_cast(delta_val)); - - assert(num_elements > 0); - - using type = decltype(start_val); - - std::vector range_vals(num_elements); - - std::generate(range_vals.begin(), range_vals.end(), [&]() { - auto result = start_val; - start_val += delta_val; - return result; - }); - - l0 = info.add_literal({shape{args[0]->get_shape().type(), {num_elements}}, range_vals}); - }); - return l0; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_reduce_op.cpp b/docker/rocm/migraphx/onnx/parse_reduce_op.cpp deleted file mode 100644 index 9d69c1b15..000000000 --- a/docker/rocm/migraphx/onnx/parse_reduce_op.cpp +++ /dev/null @@ -1,233 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -template -struct reduce_parser : op_parser -{ - instruction_ref parse_reduce_oper(const std::string& op_name, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - auto constant_axes = parse_constant_axes(args, info); - - int noop_with_empty_axes = - parse_attribute("noop_with_empty_axes", parser, info).value_or(0); - - int keep_dims = parse_attribute("keepdims", parser, info).value_or(1); - - std::vector all_axes(args.front()->get_shape().ndim()); - std::iota(all_axes.begin(), all_axes.end(), 0); - - // Handle axes attribute, constant input axes, and missing both attribute and input cases - if(constant_axes.has_value()) - { - if(noop_with_empty_axes != 0 and constant_axes->empty()) - return args[0]; - - if(noop_with_empty_axes == 0 and constant_axes->empty()) - constant_axes = all_axes; - - auto reduce = - info.add_instruction(make_op(op_name, {{"axes", *constant_axes}}), args[0]); - - if(keep_dims == 0) - return info.add_instruction(make_op("squeeze", {{"axes", *constant_axes}}), reduce); - - return reduce; - } - - // Handle variable input axes - if(keep_dims == 0) - MIGRAPHX_THROW("Keepdims not supported with runtime provided axes"); - - // Empty axes attribute indicates to the operator to look for axes in the inputs - // If the input axes are empty, the default behavior of reduce_op is to be an - // identity operator - auto reduce_op = make_op(op_name, {{"axes", {}}}); - - if(noop_with_empty_axes != 0) - return info.add_instruction(reduce_op, args); - - if(args[1]->get_shape().dynamic()) - { - auto reduce_input_axes = info.add_instruction(reduce_op, args); - auto all_axes_lit = info.add_literal( - literal{shape{shape::type_t::int64_type, {all_axes.size()}}, all_axes}); - auto reduce_all_axes = info.add_instruction(reduce_op, args[0], all_axes_lit); - auto zero = info.add_literal(literal{shape{shape::type_t::int64_type}, {0u}}); - auto axes_size = info.add_instruction(make_op("dimensions_of", {{"end", 1}}), args[1]); - auto is_axes_empty = info.add_instruction(make_op("equal"), axes_size, zero); - - return info.add_instruction( - make_op("where"), is_axes_empty, reduce_all_axes, reduce_input_axes); - } - else if(args[1]->get_shape().elements() == 0) - { - auto all_axes_lit = info.add_literal( - literal{shape{shape::type_t::int64_type, {all_axes.size()}}, all_axes}); - return info.add_instruction(reduce_op, args[0], all_axes_lit); - } - else - { - return info.add_instruction(reduce_op, args); - } - } - - private: - template - std::optional parse_attribute(const std::string& attribute_name, - const onnx_parser& parser, - onnx_parser::node_info& info) const - { - if(not contains(info.attributes, attribute_name)) - return std::nullopt; - - return parser.parse_value(info.attributes[attribute_name]).at(); - } - - std::optional> parse_constant_axes(std::vector& args, - onnx_parser::node_info& info) const - { - std::vector axes; - if(args.size() == 2) - { - if(not args[1]->can_eval()) - return std::nullopt; - args[1]->eval().visit([&](auto s) { axes.assign(s.begin(), s.end()); }); - } - else if(contains(info.attributes, "axes")) - { - auto&& attr_axes = info.attributes["axes"].ints(); - axes.assign(attr_axes.begin(), attr_axes.end()); - } - - return axes; - } -}; - -struct parse_reduce_op : reduce_parser -{ - std::vector operators() const - { - return {{"ReduceMax", "reduce_max"}, - {"ReduceMean", "reduce_mean"}, - {"ReduceMin", "reduce_min"}, - {"ReduceProd", "reduce_prod"}, - {"ReduceSum", "reduce_sum"}}; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - return parse_reduce_oper(opd.op_name, parser, std::move(info), std::move(args)); - } -}; - -struct parse_reduce_l1 : reduce_parser -{ - std::vector operators() const { return {{"ReduceL1"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - args[0] = info.add_instruction(make_op("abs"), args[0]); - return parse_reduce_oper("reduce_sum", parser, std::move(info), std::move(args)); - } -}; - -struct parse_reduce_l2 : reduce_parser -{ - std::vector operators() const { return {{"ReduceL2"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - args[0] = info.add_instruction(make_op("mul"), args[0], args[0]); - auto sum_ins = parse_reduce_oper("reduce_sum", parser, info, std::move(args)); - return info.add_instruction(make_op("sqrt"), sum_ins); - } -}; - -struct parse_reduce_log_sum : reduce_parser -{ - std::vector operators() const { return {{"ReduceLogSum"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - auto sum_ins = parse_reduce_oper("reduce_sum", parser, info, std::move(args)); - return info.add_instruction(make_op("log"), sum_ins); - } -}; - -struct parse_reduce_log_sum_exp : reduce_parser -{ - std::vector operators() const { return {{"ReduceLogSumExp"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - args[0] = info.add_instruction(make_op("exp"), args[0]); - auto sum_ins = parse_reduce_oper("reduce_sum", parser, info, std::move(args)); - return info.add_instruction(make_op("log"), sum_ins); - } -}; - -struct parse_reduce_sum_square : reduce_parser -{ - std::vector operators() const { return {{"ReduceSumSquare"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - args[0] = info.add_instruction(make_op("mul"), args[0], args[0]); - return parse_reduce_oper("reduce_sum", parser, std::move(info), std::move(args)); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_reshape.cpp b/docker/rocm/migraphx/onnx/parse_reshape.cpp deleted file mode 100644 index dba225d77..000000000 --- a/docker/rocm/migraphx/onnx/parse_reshape.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_reshape : op_parser -{ - std::vector operators() const { return {{"Reshape"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - std::vector dims; - if(args.size() == 1) - { - literal s = parser.parse_value(info.attributes.at("shape")); - s.visit([&](auto v) { copy(v, std::back_inserter(dims)); }); - return info.add_instruction(make_op("reshape", {{"dims", dims}}), args[0]); - } - else - { - // 2 inputs - auto s = args[1]->eval(); - if(s.empty()) - { - // arg[1] not eval-able - auto alloc_ins = info.add_instruction( - make_op("allocate", {{"buf_type", args[0]->get_shape().type()}}), args[1]); - return info.add_instruction(make_op("reshape"), args[0], alloc_ins); - } - else - { - s.visit([&](auto v) { copy(v, std::back_inserter(dims)); }); - return info.add_instruction(make_op("reshape", {{"dims", dims}}), args[0]); - } - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_resize.cpp b/docker/rocm/migraphx/onnx/parse_resize.cpp deleted file mode 100644 index 0c6216ae9..000000000 --- a/docker/rocm/migraphx/onnx/parse_resize.cpp +++ /dev/null @@ -1,454 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/* - * Algorithm of calc_neighbor_points(): - * Input: vvv_ind, a collection of neighbors per resized dimension as: - * layer-1: (# resized dimensions, vector) - * layer-2: (A vector of 2 of: hi/low) - * layer-3: Neighor index of every pixel in that output dimension (vector) - * in_s, the original input tensor shape (vector) - * out_s, the output tensor shape (vector) - * resized_m, lens indices that have to resized (map) - * - * Output: per resized pixel, its neighboring hi/lo indexes (vector): all permutations. - * This api stitches all the neighbors (for every dimension) for a resized pixel, - * to yield its neighbor index w.r.t to the input shape, in_s. - */ - -static std::vector -calc_neighbor_points(const std::vector>>& vvv_ind, - const shape& in_s, - const shape& out_s, - const std::map& resized_m) -{ - std::size_t ndims = out_s.ndim(); - const auto& strides = out_s.strides(); - std::size_t elements_ct = vvv_ind[0][0].size(); - - // This function computes for each element, all permutations of its neighbor indices into an - // Perm block in one go. (Instead of computing each permutation in isolation per element) - size_t permutations = 1u << resized_m.size(); - std::vector> perm_blk(permutations, std::vector(strides)); - - // final outputted vector: permutations of neighbors. - std::vector out_idx_vec(permutations * elements_ct); - - for(size_t e_idx = 0; e_idx < elements_ct; ++e_idx) - { - size_t t_idx = e_idx; - for(size_t l_idx = 0; l_idx != ndims; ++l_idx) - { - auto entry = resized_m.find(l_idx); - if(entry != resized_m.end()) - { - size_t hi_cmp_bit = 1u << entry->second; - auto lo = vvv_ind[entry->second][0][e_idx]; - auto hi = vvv_ind[entry->second][1][e_idx]; - for(size_t i = 0; i < permutations; i++) - perm_blk[i][l_idx] = ((i & hi_cmp_bit) != 0) ? hi : lo; - } - else - { - size_t idx = t_idx / strides[l_idx]; - // no permutations in an unmodified lens index, so idx is copied over: - for(size_t i = 0; i < permutations; i++) - perm_blk[i][l_idx] = idx; - } - t_idx %= strides[l_idx]; - } - // write out the permuted indices, calculated off the perm_blk: - for(size_t i = 0; i < permutations; i++) - out_idx_vec[e_idx + elements_ct * i] = in_s.index(perm_blk[i]); - } - return out_idx_vec; -} - -static std::string get_coord_trans_mode(const onnx_parser::attribute_map& attr) -{ - std::string coord_trans_mode = "half_pixel"; - if(contains(attr, "coordinate_transformation_mode")) - { - coord_trans_mode = attr.at("coordinate_transformation_mode").s(); - // does not support transformation mode "tf_crop_and_resize" - if(coord_trans_mode == "tf_crop_and_resize") - { - MIGRAPHX_THROW("PARSE_RESIZE: \"tf_crop_and_resize\" mode is not supported!"); - } - } - - return coord_trans_mode; -} - -static std::string get_mode(const onnx_parser::attribute_map& attr) -{ - std::string mode = "nearest"; - if(contains(attr, "mode")) - { - mode = attr.at("mode").s(); - if(mode != "nearest" and mode != "linear") - { - MIGRAPHX_THROW("PARSE_RESIZE: only nearest and linear modes are supported!"); - } - } - - return mode; -} - -static std::string get_nearest_mode(const onnx_parser::attribute_map& attr) -{ - std::string nearest_mode = "round_prefer_floor"; - if(contains(attr, "nearest_mode")) - { - nearest_mode = attr.at("nearest_mode").s(); - } - - return nearest_mode; -} - -// "scales" is an attribute of the deprecated Upsample op. ver7 only -static std::vector get_scales(const onnx_parser::attribute_map& attr) -{ - std::vector scales; - if(contains(attr, "scales")) - { - copy(attr.at("scales").floats(), std::back_inserter(scales)); - } - - return scales; -} - -// Hunts through the argument list to find either scales or sizes, and -// populates both scales and sizes vectors from it. -// r_arg: a reference to the argument that was found. -// -// return: true if argument is non-static (i.e. if eval() couldn't read it -// at compile time). If true, we'll need to use Resize op. -static bool parse_args(const std::vector& args, - const std::vector& in_lens, - const std::string& onnx_name, - std::vector& vec_scale, - std::vector& out_lens, - instruction_ref& r_arg) -{ - for(const auto& arg : args) - { - if(arg->name() == "undefined" or arg == args.front()) - continue; - - // skip any empty input (some of the Onnx args. are optional) - auto lens = arg->get_shape().lens(); - if(lens.empty()) - continue; - - r_arg = arg; - - auto type = arg->get_shape().type(); - if(type == shape::int64_type) - { - // this argument is output sizes - auto arg_out_s = arg->eval(); - if(arg_out_s.empty()) - return true; - arg_out_s.visit([&](const auto& ol) { out_lens.assign(ol.begin(), ol.end()); }); - - if(out_lens.size() != in_lens.size()) - { - MIGRAPHX_THROW("PARSE_" + onnx_name + - ": specified output size's rank does not match input size"); - } - - // compute the scales - vec_scale.resize(in_lens.size()); - std::transform(in_lens.begin(), - in_lens.end(), - out_lens.begin(), - vec_scale.begin(), - [](auto iss, auto oss) { return 1.0 * oss / iss; }); - return false; - } - else - { - // this argument is scale input - if(lens[0] == in_lens.size()) - { - auto arg_scale = arg->eval(); - if(arg_scale.empty()) - return true; - - arg_scale.visit([&](const auto& v) { vec_scale.assign(v.begin(), v.end()); }); - } - return false; - } - } - MIGRAPHX_THROW("PARSE_" + onnx_name + ": no shapes or scales input provided"); -} - -struct parse_resize : op_parser -{ - std::vector operators() const - { - return {{"Resize", "resize"}, {"Upsample", "upsample"}}; - } - - // Helper to add a "reshape" and "gather" instruction. These can implement - // Nearest mode resizing if all sizes are known at compile time. - instruction_ref make_gather_instruction(const onnx_parser::node_info& info, - const std::size_t out_elements, - const shape& in_s, - shape& out_s, - const std::vector& in_lens, - const std::vector& out_lens, - const std::vector& vec_scale, - instruction_ref args_0) const - { - std::string nearest_mode = get_nearest_mode(info.attributes); - std::vector ind(out_elements); - - // map out_idx to in_idx - auto nearest_op = op::resize::get_nearest_op(nearest_mode); - std::string coord_trans_mode = get_coord_trans_mode(info.attributes); - auto idx_op = op::resize::get_original_idx_op(coord_trans_mode); - - shape_for_each(out_s, [&](const auto& out_idx_v, size_t out_idx) { - std::vector in_idx(out_idx_v.size()); - for(auto ii = 0; ii < in_lens.size(); ++ii) - { - auto idx_val = idx_op(in_lens[ii], out_lens[ii], out_idx_v[ii], vec_scale[ii]); - in_idx[ii] = nearest_op(in_lens[ii], idx_val); - } - - ind[out_idx] = static_cast(in_s.index(in_idx)); - }); - // reshape input to one-dimension - std::vector rsp_lens = {static_cast(in_s.elements())}; - auto rsp = info.add_instruction(make_op("reshape", {{"dims", rsp_lens}}), args_0); - - // ins_ind should be a multi dimensional index that will restore original rank - shape ind_s{shape::int32_type, out_lens}; - auto ins_ind = info.add_literal(literal(ind_s, ind)); - return info.add_instruction(make_op("gather", {{"axis", 0}}), rsp, ins_ind); - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser&, - onnx_parser::node_info info, - std::vector args) const - { - // coord transform mode - std::string coord_trans_mode = get_coord_trans_mode(info.attributes); - - // mode: only nearest and linear modes are supported for now - std::string mode = get_mode(info.attributes); - - // nearest mode - std::string nearest_mode = get_nearest_mode(info.attributes); - - auto idx_op = op::resize::get_original_idx_op(coord_trans_mode); - - // check exclude_outside, only support 0 - if(contains(info.attributes, "exclude_outside") and - info.attributes.at("exclude_outside").i() == 1) - { - MIGRAPHX_THROW("PARSE_" + opd.onnx_name + ": exclude_outside 1 is not supported!"); - } - - // input data shape info - auto in_s = args[0]->get_shape().to_static(1); - auto in_lens = in_s.lens(); - - // output shape is explicitly specified - std::vector out_lens(in_lens.size()); - - // scale - std::vector vec_scale = get_scales(info.attributes); - - // If `scales` was not an attribute, it must be an input - // bool is_scale_input{true}; - instruction_ref scales_sizes_arg(args[0]); - - // boolean indicates whether the size of the output can be determined - // at compile time, i.e. its values come from literal input(s) and have - // no dependencies anywhere in the graph on runtime inputs. - bool is_constant_scale_input(not vec_scale.empty()); - if(not is_constant_scale_input) - { - // Depending on the args, it *must* populate the `vec_scale`, and might populate - // `out_lens` - is_constant_scale_input = - not parse_args(args, in_lens, opd.onnx_name, vec_scale, out_lens, scales_sizes_arg); - } - - if(is_constant_scale_input) - { - if(in_lens.size() != vec_scale.size()) - { - MIGRAPHX_THROW("PARSE_" + opd.onnx_name + - ": ranks of input and scale are different!"); - } - - // if the output was not calculated yet, we update it based on the scales - if(all_of(out_lens.cbegin(), out_lens.cend(), [](auto o) { return o == 0; })) - { - std::transform( - in_lens.begin(), - in_lens.end(), - vec_scale.begin(), - out_lens.begin(), - [&](auto idx, auto scale) { return static_cast(idx * scale); }); - } - } - - if(mode == "nearest") - { - if(args[0]->get_shape().dynamic() or not is_constant_scale_input) - { - // Resize's compute_shape() will read scales_sizes_arg as "scales" or "sizes" - // depending on its data type - return info.add_instruction( - make_op("resize", - {{"nearest_mode", nearest_mode}, - {"coordinate_transformation_mode", coord_trans_mode}}), - args[0], - scales_sizes_arg); - } - else - { - // If there are no dynamic shapes and size/scale attributes are literals, then - // all the indexes can be calculated now at compile time and - // the Resize can be accomplished with Gather operation. Preferred for - // better performance. - - shape out_s{in_s.type(), out_lens}; - std::size_t out_elements = out_s.elements(); - - return make_gather_instruction( - info, out_elements, in_s, out_s, in_lens, out_lens, vec_scale, args[0]); - } - } - // linear mode - else - { - // out_lens and other variables can't be populated if non-constant (runtime) size - // inputs. - if(not is_constant_scale_input) - MIGRAPHX_THROW("PARSE_" + opd.onnx_name + - ": linear mode not supported for non-constant inputs"); - - shape out_s{in_s.type(), out_lens}; - - // reshape input to one-dimension - std::vector rsp_lens = {static_cast(in_s.elements())}; - auto rsp = info.add_instruction(make_op("reshape", {{"dims", rsp_lens}}), args[0]); - - auto nearest_floor = op::resize::get_nearest_op("floor"); - auto nearest_ceil = op::resize::get_nearest_op("ceil"); - - std::vector resized_axes; // vector of dimensions to be resized - std::size_t out_elements = 1; // total number of elements to be resized - size_t resized_ct = 0; - std::map resized_m; // modified indices --> vvv_ind index below - for(std::size_t axis = 0; axis != out_lens.size(); ++axis) - { - out_elements *= out_lens[axis]; - if(in_lens[axis] == out_lens[axis]) - continue; - resized_axes.push_back(axis); - resized_m[axis] = resized_ct++; - } - - // Neighbor indices. For an axis. Two sets of max/min per element: - std::vector> vv_ind(2, std::vector(out_elements)); - // Neighbor indices. For all resized axes: - std::vector>> vvv_ind(resized_ct, vv_ind); - // Delta list. For each resized axes - per element. - std::vector> delta(resized_ct, std::vector(out_elements)); - - shape_for_each(out_s, [&](const auto& out_idx_v, std::size_t out_idx) { - for(size_t ii = 0; ii != resized_ct; ++ii) - { - auto idx = resized_axes[ii]; - auto idx_val = - idx_op(in_lens[idx], out_lens[idx], out_idx_v[idx], vec_scale[idx]); - vvv_ind[ii][0][out_idx] = nearest_floor(in_lens[idx], idx_val); - vvv_ind[ii][1][out_idx] = nearest_ceil(in_lens[idx], idx_val); - delta[ii][out_idx] = idx_val - vvv_ind[ii][0][out_idx]; - } - }); - - auto ind = calc_neighbor_points(vvv_ind, in_s, out_s, resized_m); - - auto dim_lens = out_lens; - // indices matrix size grows 2x per resized-axis: - dim_lens[0] *= (1u << resized_ct); - shape ind_s{shape::int32_type, dim_lens}; - auto ins_ind = info.add_literal(literal(ind_s, ind)); - auto data = info.add_instruction(make_op("gather", {{"axis", 0}}), rsp, ins_ind); - - for(auto idx = resized_ct; idx != 0u; --idx) - { - dim_lens[0] /= 2; // halved for 2 slices of data (hi & low below) - shape dim_s{shape::float_type, dim_lens}; - const auto& dim_delta = delta[idx - 1]; - std::vector delta_data; - for(std::size_t j = 0; j < dim_lens[0] / out_lens[0]; ++j) - delta_data.insert(delta_data.begin(), dim_delta.begin(), dim_delta.end()); - auto ins_delta = info.add_literal(dim_s, delta_data); - - // slice the data - int64_t slc_stride = dim_lens[0]; - auto low = info.add_instruction( - make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {slc_stride}}}), - data); - auto hi = info.add_instruction( - make_op("slice", - {{"axes", {0}}, {"starts", {slc_stride}}, {"ends", {2 * slc_stride}}}), - data); - auto diff = info.add_instruction(make_op("sub"), hi, low); - auto ddf = info.add_instruction(make_op("mul"), diff, ins_delta); - data = info.add_instruction(make_op("add"), ddf, low); - } - return data; - } - } -}; - -} // namespace onnx - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_reversesequence.cpp b/docker/rocm/migraphx/onnx/parse_reversesequence.cpp deleted file mode 100644 index cb81695e4..000000000 --- a/docker/rocm/migraphx/onnx/parse_reversesequence.cpp +++ /dev/null @@ -1,148 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// Parser for ReverseSequence ONNX operator. -/*! - Reverses the data along the time axis for the batches along the batch axis. - The sequence lengths can be given to reverse up to the given length for each batch, keeping the - rest of the sequence in the original order. Variable sequence_lens is not supported in this - version of MIGraphX. You can pass the sequence_lens either as a constant node or an attribute. The - batch axis and time axis must be [0, 1] and not the same. -*/ -struct parse_reversesequence : op_parser -{ - std::vector operators() const { return {{"ReverseSequence"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - int batch_axis = 1; - if(contains(info.attributes, "batch_axis")) - { - batch_axis = info.attributes.at("batch_axis").i(); - } - if(batch_axis != 0 and batch_axis != 1) - { - MIGRAPHX_THROW("REVERSESEQUENCE: batch axis not 0 or 1"); - } - - int time_axis = 0; - if(contains(info.attributes, "time_axis")) - { - time_axis = info.attributes.at("time_axis").i(); - } - if(time_axis != 0 and time_axis != 1) - { - MIGRAPHX_THROW("REVERSESEQUENCE: time axis not 0 or 1"); - } - - if(time_axis == batch_axis) - { - MIGRAPHX_THROW("REVERSESEQUENCE: time axis and batch axis are the same"); - } - - auto input = args[0]; - auto input_lens = input->get_shape().lens(); - if(input_lens.size() < 2) - { - MIGRAPHX_THROW("REVERSESEQUENCE: input tensor must have rank >= 2"); - } - - std::vector sequence_lens; - if(args.size() == 2) - { - migraphx::argument seq_lens_arg = args.back()->eval(); - check_arg_empty(seq_lens_arg, "REVERSESEQUENCE: cannot handle variable sequence_lens"); - seq_lens_arg.visit([&](auto s) { sequence_lens.assign(s.begin(), s.end()); }); - } - else if(contains(info.attributes, "sequence_lens")) - { - literal s = parser.parse_value(info.attributes.at("sequence_lens")); - s.visit([&](auto v) { sequence_lens.assign(v.begin(), v.end()); }); - } - auto batch_size = input_lens[batch_axis]; - auto time_size = input_lens[time_axis]; - - // this condition may still work if sequence_len's shape was incorrect - if(sequence_lens.size() != batch_size) - { - MIGRAPHX_THROW("REVERSESEQUENCE: sequence_lens has incorrect shape"); - } - - instruction_ref ret; - - auto add_slice = [&info, &input, batch_axis, time_axis](int b, int t_start, int t_end) { - return info.add_instruction(make_op("slice", - {{"axes", {batch_axis, time_axis}}, - {"starts", {b, t_start}}, - {"ends", {b + 1, t_end}}}), - input); - }; - - for(int b = 0; b < batch_size; ++b) - { - instruction_ref s0; - if(sequence_lens[b] > 1) - { - s0 = add_slice(b, 0, sequence_lens[b]); - s0 = info.add_instruction(make_op("reverse", {{"axes", {time_axis}}}), s0); - - // if reversed less than whole batch, concat rest of batch - if(sequence_lens[b] < time_size) - { - auto s1 = add_slice(b, sequence_lens[b], time_size); - s0 = info.add_instruction(make_op("concat", {{"axis", time_axis}}), s0, s1); - } - } - else - { // cases where nothing changes - s0 = add_slice(b, 0, time_size); - } - if(b == 0) - { - ret = s0; - } - else - { - ret = info.add_instruction(make_op("concat", {{"axis", batch_axis}}), ret, s0); - } - } - return ret; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_rnn.cpp b/docker/rocm/migraphx/onnx/parse_rnn.cpp deleted file mode 100644 index 9f669510f..000000000 --- a/docker/rocm/migraphx/onnx/parse_rnn.cpp +++ /dev/null @@ -1,185 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -void rnn_transpose_inputs(onnx_parser::node_info& info, std::vector& args) -{ - std::vector perm{1, 0, 2}; - args[0] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[0]); - - if(not args[5]->is_undefined()) - { - args[5] = info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[5]); - } -} - -void rnn_transpose_outputs(onnx_parser::node_info& info, - instruction_ref& hidden_states, - instruction_ref& last_output) -{ - std::vector perm_hs{2, 0, 1, 3}; - hidden_states = - info.add_instruction(make_op("transpose", {{"permutation", perm_hs}}), hidden_states); - std::vector perm_last{1, 0, 2}; - last_output = - info.add_instruction(make_op("transpose", {{"permutation", perm_last}}), last_output); -} - -struct parse_rnn : op_parser -{ - std::vector operators() const { return {{"RNN"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - migraphx::shape input_shape = args[0]->get_shape(); - std::size_t hidden_size = args[1]->get_shape().lens()[1]; - - if(contains(info.attributes, "hidden_size")) - { - std::size_t hidden_size_att = - parser.parse_value(info.attributes.at("hidden_size")).at(); - if(hidden_size != hidden_size_att) - { - MIGRAPHX_THROW("RNN: hidden size mismatch in input and attribute"); - } - } - - // Handling of direction to be added later - std::string direction{"forward"}; - if(contains(info.attributes, "direction")) - { - direction = info.attributes.at("direction").s(); - } - - op::rnn_direction dirct = op::rnn_direction::forward; - if(direction == "bidirectional") - { - dirct = op::rnn_direction::bidirectional; - } - else if(direction == "reverse") - { - dirct = op::rnn_direction::reverse; - } - - std::vector vec_names = {"tanh", "tanh"}; - if(contains(info.attributes, "activations")) - { - auto names = info.attributes.at("activations").strings(); - vec_names.clear(); - vec_names.resize(names.size()); - std::transform(names.begin(), names.end(), vec_names.begin(), [](auto name) { - return to_lower(name); - }); - } - - if(vec_names.size() == 2 and dirct != op::rnn_direction::bidirectional) - { - // default activations are {"tanh", "tanh"} - // in this case, take the first default activation. - vec_names.resize(1); - } - - auto num_actv_functions = dirct == op::rnn_direction::bidirectional ? 2 : 1; - if(vec_names.size() != static_cast(num_actv_functions)) - { - MIGRAPHX_THROW("RNN: Invalid activation functions number, should be: " + - to_string(num_actv_functions)); - } - - auto name_it = std::find_if(vec_names.begin(), vec_names.end(), [&](auto& name) { - return (map_activation_functions().count(name) == 0); - }); - if(name_it != vec_names.end()) - { - MIGRAPHX_THROW("RNN: activation function " + std::string(*name_it) + " not supported"); - } - - std::vector vec_actv_funcs(vec_names.size()); - std::transform(vec_names.begin(), - vec_names.end(), - vec_actv_funcs.begin(), - [&](const auto& fn) { return map_activation_functions().at(fn); }); - - // To be added later - float clip = 0.0; - if(contains(info.attributes, "clip")) - { - clip = parser.parse_value(info.attributes.at("clip")).at(); - } - - int layout = 0; - if(contains(info.attributes, "layout")) - { - layout = parser.parse_value(info.attributes.at("layout")).at(); - } - - // if the number of arguments is less than 6, append - // undefined operator to have 6 arguments - if(args.size() < 6) - { - auto ins = info.add_instruction(make_op("undefined")); - args.insert(args.end(), (6 - args.size()), ins); - } - - if(layout != 0) - { - rnn_transpose_inputs(info, args); - } - - // first output for the concatenation of hidden states - auto hidden_states = info.add_instruction(make_op("rnn", - {{"hidden_size", hidden_size}, - {"actv_func", to_value(vec_actv_funcs)}, - {"direction", dirct}, - {"clip", clip}}), - args); - - // second output for the last hidden state - auto last_output = info.add_instruction(make_op("rnn_last_hs_output"), hidden_states); - - if(layout != 0) - { - rnn_transpose_outputs(info, hidden_states, last_output); - } - - return {hidden_states, last_output}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_roialign.cpp b/docker/rocm/migraphx/onnx/parse_roialign.cpp deleted file mode 100644 index 7db57a71a..000000000 --- a/docker/rocm/migraphx/onnx/parse_roialign.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_roialign : op_parser -{ - std::vector operators() const { return {{"RoiAlign"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - const std::vector& args) const - { - std::string coord_trans_mode = - parser.opset_version >= 16 ? "half_pixel" : "output_half_pixel"; - - if(const auto* a = "coordinate_transformation_mode"; contains(info.attributes, a)) - { - coord_trans_mode = info.attributes.at(a).s(); - } - - if(not contains({"half_pixel", "output_half_pixel"}, coord_trans_mode)) - { - MIGRAPHX_THROW("coordinate_transformation_mode \"" + coord_trans_mode + - "\": invalid value!"); - } - - migraphx::op::pooling_mode rmode(migraphx::op::pooling_mode::average); - if(contains(info.attributes, "mode")) - { - // read mode; default is "avg" - if(info.attributes.at("mode").s() == "max") - { - rmode = migraphx::op::pooling_mode::max; - } - } - - int64_t output_height = 1; - if(contains(info.attributes, "output_height")) - { - output_height = info.attributes.at("output_height").i(); - } - - int64_t output_width = 1; - if(contains(info.attributes, "output_width")) - { - output_width = info.attributes.at("output_width").i(); - } - - int64_t sampling_ratio = 0; - if(contains(info.attributes, "sampling_ratio")) - { - sampling_ratio = info.attributes.at("sampling_ratio").i(); - } - - float spatial_scale = 1.0f; - if(contains(info.attributes, "spatial_scale")) - { - spatial_scale = info.attributes.at("spatial_scale").f(); - } - return info.add_instruction(make_op("roialign", - {{"coordinate_transformation_mode", coord_trans_mode}, - {"mode", rmode}, - {"output_height", output_height}, - {"output_width", output_width}, - {"sampling_ratio", sampling_ratio}, - {"spatial_scale", spatial_scale}}), - args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_scan.cpp b/docker/rocm/migraphx/onnx/parse_scan.cpp deleted file mode 100644 index 15db9e1f4..000000000 --- a/docker/rocm/migraphx/onnx/parse_scan.cpp +++ /dev/null @@ -1,324 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_scan : op_parser -{ - std::vector operators() const { return {{"Scan"}}; } - - std::vector parse(const op_desc& /*opd*/, - onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - if(parser.opset_version == 8) - MIGRAPHX_THROW("Scan: Opset 8 version not supported"); - - check_for_required_attributes(info, {"body", "num_scan_inputs"}); - - const auto& body_graph = info.attributes["body"].g(); - auto* body = parser.prog.create_module(info.name + "_scan"); - parser.parse_graph(body, body_graph); - - // Scan has: - // N + M inputs (N state variables, M scan inputs) - // N + K outputs (N state variables, K scan outputs) - // Same input and output counts apply for body - auto body_outs = body->get_returns(); - const auto m = info.attributes["num_scan_inputs"].i(); - const auto n = args.size() - m; - const auto k = body_outs.size() - n; - - std::vector body_params; - transform(body->get_parameter_names(), - std::back_inserter(body_params), - [&](const auto& name) { return body->get_parameter(name); }); - - if(auto num_body_params = body_params.size(); num_body_params != n + m) - MIGRAPHX_THROW("Scan: Number of inputs to body {" + std::to_string(num_body_params) + - "} does not match number of inputs to Scan {" + std::to_string(n + m) + - "}"); - - const auto scan_input_axes = parse_axes(info, "scan_input_axes", m, args.begin() + n, 0); - const auto scan_input_directions = parse_dirs(info, "scan_input_directions", m); - const auto scan_output_axes = - parse_axes(info, "scan_output_axes", k, body_outs.begin() + n, 1); - const auto scan_output_directions = parse_dirs(info, "scan_output_directions", k); - - // Check that scan axes lens are the same across all scan inputs - size_t num_iters = args[n]->get_shape().lens()[scan_input_axes[0]]; - for(auto i = 1; i < m; ++i) - if(args[n + i]->get_shape().lens()[scan_input_axes[i]] != num_iters) - MIGRAPHX_THROW( - "Scan: Lengths of scan_input_axes do not match across all scan inputs.\n" - "Scan input shapes: " + - to_string_range( - to_shapes(std::vector(args.begin() + n, args.end()))) + - "\nScan input axes: " + to_string_range(scan_input_axes)); - - if(num_iters > parser.max_loop_iterations) - MIGRAPHX_THROW("Scan: Number of required iterations {" + std::to_string(num_iters) + - "} would exceed the maximum iteration limit {" + - std::to_string(parser.max_loop_iterations) + "}"); - - // Check that state variable shapes match between the Scan node and its body attribute - for(auto i = 0; i < n; ++i) - if(args[i]->get_shape() != body_params[i]->get_shape()) - MIGRAPHX_THROW("Scan: State input " + std::to_string(i) + " shape {" + - to_string(args[i]->get_shape()) + - "} does not match corresponding body input shape {" + - to_string(body_params[i]->get_shape()) + "}"); - - // Check that the shapes of scan inputs sliced across scan input axes match the shapes of - // the body attribute scan inputs - for(auto i = 0; i < m; ++i) - { - auto node_shape = args[i + n]->get_shape(); - auto node_lens = node_shape.lens(); - node_lens.erase(node_lens.begin() + scan_input_axes[i]); - auto slice_sh = shape(node_shape.type(), std::move(node_lens)); - if(body_params[i + n]->get_shape() != slice_sh) - MIGRAPHX_THROW("Slice: Sliced scan input " + std::to_string(i) + " shape {" + - to_string(slice_sh) + - "} does not match corresponding body input shape {" + - to_string(body_params[i + n]->get_shape()) + "}"); - } - - modify_body(body, args, n, m, scan_input_axes, scan_input_directions); - - auto max_iter_lit = info.add_literal(literal{shape{shape::int64_type}, {num_iters}}); - auto cond_lit = info.add_literal(literal{shape{shape::bool_type}, {true}}); - std::vector loop_args{max_iter_lit, cond_lit}; - loop_args.insert(loop_args.end(), args.begin(), args.begin() + n); - - auto loop = - info.add_instruction(make_op("loop", - {{"max_iterations", num_iters}, - {"scan_output_directions", scan_output_directions}}), - loop_args, - {body}); - - std::vector ret; - ret.reserve(n + k); - for(auto i = 0; i < n; ++i) - ret.push_back(info.add_instruction(make_op("get_tuple_elem", {{"index", i}}), loop)); - - for(auto i = 0; i < k; ++i) - { - auto o = info.add_instruction(make_op("get_tuple_elem", {{"index", i + n}}), loop); - // Loop concatenates scan axes along axis 0 which is inserted/unsqueezed, e.g. a body - // scan output(from a single iteration) of shape {2, 2} is first expanded to {1, 2, 2}, - // and then concatenated with body scan outputs from previous iterations. For n - // iterations of the loop, this will end up producing a scan output of shape {n, 2, 2}. - // - // The scan_output_axes attribute of Scan can define an axis other than zero as the - // concatenation axis. Using the previous scenario, for a body scan output of - // shape {2,2}, with the scan output axis being 1, it is unsqueezed to {2, 1, 2}. The - // final concatenation is then of shape {2, n, 2}. - // - // Since Loop only concatenates along the unsqueezed axis 0, a transpose is necessary to - // place axis 0 in the appropriate scan_output_axis position - auto perm = make_perm_for_scan_out(o->get_shape().ndim(), scan_output_axes[i]); - ret.push_back(info.add_instruction(make_op("transpose", {{"permutation", perm}}), o)); - } - - return ret; - } - - void check_for_required_attributes(onnx_parser::node_info& info, - const std::vector& attribute_names) const - { - auto it = std::find_if( - attribute_names.cbegin(), attribute_names.cend(), [&](const std::string& name) { - return not contains(info.attributes, name); - }); - if(it != attribute_names.cend()) - MIGRAPHX_THROW("Scan: " + *it + " attribute required"); - } - - std::vector parse_vector_attribute(onnx_parser::node_info& info, - const std::string& attr_name, - size_t expected_size) const - { - if(not contains(info.attributes, attr_name)) - return {}; - - std::vector res; - auto&& attr = info.attributes[attr_name].ints(); - if(attr.size() != expected_size) - MIGRAPHX_THROW("Scan: " + attr_name + " size is " + to_string(attr.size()) + - ", should be " + to_string(expected_size)); - res.assign(attr.begin(), attr.end()); - - return res; - } - - std::vector - parse_dirs(onnx_parser::node_info& info, const std::string& name, size_t expected_size) const - { - auto dirs = parse_vector_attribute(info, name, expected_size); - if(dirs.empty()) - return std::vector(expected_size, 0); // NOLINT - - if(any_of(dirs, [](auto i) { return i != 0 and i != 1; })) - MIGRAPHX_THROW("Scan: " + name + - " may contain only 1s and 0s, actual values: " + to_string_range(dirs)); - - return dirs; - } - - int64_t normalize_axis(int64_t axis, int64_t rank, const std::string& attr_name) const - { - if(axis < -rank or axis >= rank) - MIGRAPHX_THROW("Scan: " + attr_name + " axis value {" + to_string(axis) + - "} out of range [" + to_string(-rank) + ", " + to_string(rank) + ")"); - - return axis < 0 ? rank + axis : axis; - } - - std::vector parse_axes(onnx_parser::node_info& info, - const std::string& name, - long expected_size, - std::vector::iterator ins_begin, - size_t rank_offset) const - { - auto axes = parse_vector_attribute(info, name, expected_size); - if(axes.empty()) - return std::vector(expected_size, 0); // NOLINT - - std::transform(axes.begin(), - axes.end(), - ins_begin, - axes.begin(), - [&](int64_t axis, instruction_ref arg) { - return normalize_axis(axis, arg->get_shape().ndim() + rank_offset, name); - }); - - return axes; - } - - // Alter the Scan body to match a body that Loop would expect. - // - // Loop body inputs: iteration_num, condition, loop_state_variables - // Scan body inputs: loop_state_variables, scan_input_slices - // iteration_num and condition parameters are prepended to the Scan body parameter list, while - // scan_input_slices are removed from parameters. - // Instead, scan_inputs are used directly in Scan body(as values from enclosing scope), and - // together with iteration_num passed to the scan_slice operator which produces slices that are - // used instead of the scan_inputs_slices. - // - // Loop body outputs: condition, loop_state_variables, scan_output_slices - // Scan body outputs: loop_state_variables, scan_output_slices - // The inserted Scan body condition parameter is prepended to the Scan body returns - void modify_body(module_ref mod, - const std::vector& args, - int64_t n, - int64_t m, - const std::vector& scan_input_axes, - const std::vector& scan_input_directions) const - { - std::vector params; - params.reserve(n + m); - transform(mod->get_parameter_names(), - std::back_inserter(params), - [&](const std::string& name) { return mod->get_parameter(name); }); - - // iteration_num, condition, and duplicate loop_state_variables are appended to parameters. - // References to the original loop_state_variables in other instructions are then replaced - // with references to the duplicate ones, after which the originals are removed. - // - // References to the scan_input_slices are replaced with references to inserted - // scan_slice->squeeze instructions, after which the scan_input_slices parameters are - // removed. - auto iter_param = mod->add_parameter("iter", shape{shape::int64_type}); - auto cond_param = mod->add_parameter("cond", shape{shape::bool_type}); - std::vector new_params; - new_params.reserve(n); - for(auto i = 0; i < n; ++i) - new_params.push_back( - mod->add_parameter("state_var" + std::to_string(i), params[i]->get_shape())); - - for(auto i = 0; i < params.size(); ++i) - { - if(i < n) - { - mod->replace_instruction(params[i], new_params[i]); - } - else - { - auto scan_axis = scan_input_axes[i - n]; - auto scan_dir = scan_input_directions[i - n]; - auto new_ins = mod->insert_instruction( - params[i], - make_op("scan_slice", {{"axis", scan_axis}, {"direction", scan_dir}}), - args[i], - iter_param); - new_ins = mod->insert_instruction( - params[i], make_op("squeeze", {{"axes", {scan_axis}}}), new_ins); - mod->replace_instruction(params[i], new_ins); - } - mod->remove_instruction(params[i]); - } - - auto returns = mod->get_returns(); - returns.insert(returns.begin(), cond_param); - mod->replace_return(returns); - } - - // Creates permutation so that axis 0 will be permuted to position axis, while maintaining the - // relative ordering of all the other axes. - // e.g. for rank = 4, axis = 2, the created perm is: [1, 2, 0, 3] - std::vector make_perm_for_scan_out(int64_t rank, int64_t axis) const - { - std::vector perm(rank); - std::iota(perm.begin(), perm.end(), 0); - std::copy(perm.begin() + 1, perm.begin() + 1 + axis, perm.begin()); - perm[axis] = 0; - - return perm; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_scatter.cpp b/docker/rocm/migraphx/onnx/parse_scatter.cpp deleted file mode 100644 index 54741b3e1..000000000 --- a/docker/rocm/migraphx/onnx/parse_scatter.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_scatter : op_parser -{ - std::vector operators() const { return {{"ScatterElements"}, {"Scatter"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - const std::vector& args) const - { - operation op; - - std::string reduction = "none"; - int axis = 0; - - if(contains(info.attributes, "axis")) - axis = info.attributes.at("axis").i(); - - if(contains(info.attributes, "reduction")) - { - reduction = info.attributes.at("reduction").s(); - // check for a valid reduction attribute. We have an operator for each one. - if(not contains({"none", "add", "mul", "min", "max"}, reduction)) - MIGRAPHX_THROW("PARSE_SCATTER: unsupported reduction mode " + reduction); - // merge scatter with reduction attribute to specify which scatter operation. Future - // reduction op names should follow this pattern and should also be added to the check - // above. - } - op = migraphx::make_op("scatter_" + reduction, {{"axis", axis}}); - - return info.add_instruction(op, args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_scatternd.cpp b/docker/rocm/migraphx/onnx/parse_scatternd.cpp deleted file mode 100644 index b088d645e..000000000 --- a/docker/rocm/migraphx/onnx/parse_scatternd.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_scatternd : op_parser -{ - std::vector operators() const { return {{"ScatterND"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector& args) const - { - std::string reduction = "none"; - if(contains(info.attributes, "reduction")) - { - reduction = info.attributes.at("reduction").s(); - if(not contains({"none", "add", "mul", "min", "max"}, reduction)) - { - MIGRAPHX_THROW("PARSE_SCATTERND: unsupported reduction mode " + reduction); - } - } - - return info.add_instruction(migraphx::make_op("scatternd_" + reduction), args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_selu.cpp b/docker/rocm/migraphx/onnx/parse_selu.cpp deleted file mode 100644 index c263469f0..000000000 --- a/docker/rocm/migraphx/onnx/parse_selu.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_selu : op_parser -{ - std::vector operators() const { return {{"Selu"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - auto type = args[0]->get_shape().type(); - auto lens = args[0]->get_shape().lens(); - float alpha = 1.67326f; - if(contains(info.attributes, "alpha")) - { - alpha = info.attributes.at("alpha").f(); - } - - float gamma = 1.0507f; - if(contains(info.attributes, "gamma")) - { - gamma = info.attributes.at("gamma").f(); - } - - auto l_alpha = info.add_literal({{type, {1}}, {alpha}}); - auto l_gamma = info.add_literal({{type, {1}}, {gamma / 2.0f}}); - if(lens != std::vector{1}) - { - l_alpha = - info.add_instruction(make_op("multibroadcast", {{"out_lens", lens}}), l_alpha); - l_gamma = - info.add_instruction(make_op("multibroadcast", {{"out_lens", lens}}), l_gamma); - } - - auto sign_x = info.add_instruction(make_op("sign"), args[0]); - auto exp_x = info.add_instruction(make_op("exp"), args[0]); - - auto alpha_ex = info.add_instruction(make_op("mul"), l_alpha, exp_x); - auto aex_alpha = info.add_instruction(make_op("sub"), alpha_ex, l_alpha); - - auto ins1 = info.add_instruction(make_op("add"), aex_alpha, args[0]); - auto ins2 = info.add_instruction(make_op("sub"), aex_alpha, args[0]); - - auto sign2 = info.add_instruction(make_op("mul"), sign_x, ins2); - auto ins_sub = info.add_instruction(make_op("sub"), ins1, sign2); - - return info.add_instruction(make_op("mul"), ins_sub, l_gamma); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_shape.cpp b/docker/rocm/migraphx/onnx/parse_shape.cpp deleted file mode 100644 index 38fa28b27..000000000 --- a/docker/rocm/migraphx/onnx/parse_shape.cpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -/** - * If static shape input, creates a literal in migraphx. - * If dynamic shape input, creates a dimensions_of operator in migraphx (runtime evaluation of - * shape). - */ -struct parse_shape : op_parser -{ - std::vector operators() const { return {{"Shape"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - if(args.size() != 1) - MIGRAPHX_THROW("Shape: operator should have 1 operand"); - auto input_shape = args[0]->get_shape(); - int input_ndim = input_shape.ndim(); - std::size_t start = 0; - std::size_t end = input_ndim; - // Normalizing the start and end is handled here because of how the static shape version - // works. Clamping to [-r, r], where r is ndim of input and then making positive. - auto normalize_ind = [&](int64_t ind) { - if(ind < (-1 * input_ndim)) - { - ind = -1 * input_ndim; - } - if(ind > input_ndim) - { - ind = input_ndim; - } - return (ind >= 0) ? ind : input_ndim + ind; - }; - if(contains(info.attributes, "end")) - { - end = normalize_ind(info.attributes.at("end").i()); - } - if(contains(info.attributes, "start")) - { - start = normalize_ind(info.attributes.at("start").i()); - } - if(end <= start) - { - MIGRAPHX_THROW("PARSE_SHAPE: ending axis <= starting axis, end: " + - std::to_string(end) + " start: " + std::to_string(start)); - } - - if(input_shape.dynamic()) - { - return info.add_instruction(make_op("dimensions_of", {{"start", start}, {"end", end}}), - args[0]); - } - else - { - std::size_t output_ndim = end - start; - std::vector vec_shape(output_ndim); - migraphx::shape s(migraphx::shape::int64_type, {output_ndim}); - std::vector input_lens = input_shape.lens(); - std::transform(input_lens.begin() + start, - input_lens.begin() + end, - vec_shape.begin(), - [](auto i) { return int64_t(i); }); - return info.add_literal(migraphx::literal{s, vec_shape}); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_shrink.cpp b/docker/rocm/migraphx/onnx/parse_shrink.cpp deleted file mode 100644 index 6fed900ef..000000000 --- a/docker/rocm/migraphx/onnx/parse_shrink.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_shrink : op_parser -{ - std::vector operators() const { return {{"Shrink"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - float bias = 0.0; - if(contains(info.attributes, "bias")) - { - bias = parser.parse_value(info.attributes.at("bias")).at(); - } - float lambd = 0.5; - if(contains(info.attributes, "lambd")) - { - lambd = parser.parse_value(info.attributes.at("lambd")).at(); - } - - auto x = args[0]; - auto x_shape = x->get_shape(); - auto x_type = x_shape.type(); - auto lit_bias = info.add_literal(bias); - auto lit_neg_lambd = info.add_literal(-lambd); - auto lit_lambd = info.add_literal(lambd); - - auto x_plus_bias = info.add_common_op("add", x, lit_bias); - auto x_min_bias = info.add_common_op("sub", x, lit_bias); - - auto cond1 = info.add_common_op("less", x, lit_neg_lambd); - auto cond2_a = info.add_common_op("not", cond1); - auto cond2_b = info.add_common_op("greater", x, lit_lambd); - auto cond2 = info.add_common_op("logical_and", cond2_a, cond2_b); - - auto first = info.add_common_op("mul", cond1, x_plus_bias); - auto second = info.add_common_op("mul", cond2, x_min_bias); - auto ret = info.add_common_op("add", first, second); - if(ret->get_shape().type() != x_type) - { - ret = info.add_instruction(make_op("convert", {{"target_type", x_type}}), ret); - } - return ret; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_simplified_layer_normalization.cpp b/docker/rocm/migraphx/onnx/parse_simplified_layer_normalization.cpp deleted file mode 100644 index 11f9d316e..000000000 --- a/docker/rocm/migraphx/onnx/parse_simplified_layer_normalization.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// ONNXRunTime implementation for reference: -// https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/core/providers/cpu/nn/layer_norm_impl.cc - -struct parse_simplified_layer_normalization : op_parser -{ - std::vector operators() const { return {{"SimplifiedLayerNormalization"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - int64_t axis = -1; - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - float epsilon = 1e-5f; - if(contains(info.attributes, "epsilon")) - { - epsilon = parser.parse_value(info.attributes.at("epsilon")).at(); - } - if(contains(info.attributes, "stash_type")) - { - std::cerr << "WARNING: SIMPLIFIED_LAYER_NORMALIZATION attribute stash_type is only " - "used for training.\n"; - } - - if(args.size() != 2) - { - MIGRAPHX_THROW( - "PARSE_SIMPLIFIED_LAYER_NORMALIZATION: invalid input count - expected 2 got " + - std::to_string(args.size())); - } - - auto x = args.at(0); - auto scale = args.at(1); - - auto x_shape = x->get_shape(); - auto x_dtype = x_shape.type(); - int64_t x_rank = x_shape.ndim(); - axis = axis < 0 ? axis + x_rank : axis; - - if(x_rank < 2 or x_rank > 3) - { - MIGRAPHX_THROW("PARSE_SIMPLIFIED_LAYER_NORMALIZATION: invalid input shape"); - } - - // Convert to float before reduce_mean - // Fp16 reduce_mean on GPU causes loss of accuracy - auto float_x = info.add_instruction( - make_op("convert", {{"target_type", migraphx::shape::float_type}}), x); - auto x_sq = info.add_common_op("mul", float_x, float_x); - auto rms = info.add_instruction(make_op("reduce_mean", {{"axes", {axis}}}), x_sq); - rms = info.add_instruction(make_op("convert", {{"target_type", x_dtype}}), rms); - auto mean = rms; - epsilon = - (x_dtype == migraphx::shape::half_type and std::abs(epsilon) < 1e-7) ? 1e-7 : epsilon; - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_dtype}, {epsilon}}); - rms = info.add_common_op("add", rms, eps); - auto rrms = info.add_instruction(make_op("rsqrt"), rms); - auto result = info.add_common_op("mul", x, rrms); - result = info.add_common_op("mul", result, scale); - - return {result, mean, rrms}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_size.cpp b/docker/rocm/migraphx/onnx/parse_size.cpp deleted file mode 100644 index 0f59f4d27..000000000 --- a/docker/rocm/migraphx/onnx/parse_size.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_size : op_parser -{ - std::vector operators() const { return {{"Size"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser&, - const onnx_parser::node_info& info, - std::vector args) const - { - return info.add_literal(migraphx::literal{migraphx::shape{migraphx::shape::int64_type}, - {args[0]->get_shape().elements()}}); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_skip_simplified_layer_normalization.cpp b/docker/rocm/migraphx/onnx/parse_skip_simplified_layer_normalization.cpp deleted file mode 100644 index 6a9d377e2..000000000 --- a/docker/rocm/migraphx/onnx/parse_skip_simplified_layer_normalization.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// com.microsoft.SkipSimplifiedLayerNormalization -// Skip and Root Mean Square Layer Normalization - -// Version -// This version of the operator has been available since version 1 of the 'com.microsoft' operator -// set. - -// Type Constraints -// T : tensor(float), tensor(float16) -// Constrain input and output types to float or half tensors. -// U : tensor(float) -// Constrain mean and inv_std_var to float tensors. - -struct parse_skip_simplified_layer_normalization - : op_parser -{ - std::vector operators() const { return {{"SkipSimplifiedLayerNormalization"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - // Attributes - // epsilon : float - // The epsilon value to use to avoid division by zero. - float epsilon = 1e-5f; - if(contains(info.attributes, "epsilon")) - { - epsilon = parser.parse_value(info.attributes.at("epsilon")).at(); - } - - // Inputs (3 - 4) - // input : T - // 3D input tensor with shape (batch_size, sequence_length, hidden_size) Or 2D input tensor - // with shape (token_count, hidden_size) - // skip : T - // 3D input tensor with shape (batch_size, sequence_length, hidden_size) - // Or 2D input tensor with shape (token_count, hidden_size) - // gamma : T - // 1D input tensor with shape (hidden_size) - // bias (optional) : T - // 1D bias tensor with shape (hidden_size) - not used by ORT - - if(args.size() < 3 or args.size() > 4) - { - MIGRAPHX_THROW("PARSE_SKIPSIMPLIFIEDLAYERNORMALIZATION: invalid input count"); - } - - auto x = args.at(0); - auto skip = args.at(1); - auto gamma = args.at(2); - instruction_ref bias; - if(args.size() == 4) - { - bias = args.at(3); - } - - auto x_shape = x->get_shape(); - auto x_dtype = x_shape.type(); - int64_t x_rank = x_shape.ndim(); - int64_t skip_rank = skip->get_shape().ndim(); - int64_t gamma_rank = gamma->get_shape().ndim(); - // axis = hidden_size dim - int64_t axis = x_rank - 1; - - if(x_rank < 2 or x_rank > 3 or x_rank != skip_rank or gamma_rank != 1) - { - MIGRAPHX_THROW("PARSE_SKIPSIMPLIFIEDLAYERNORMALIZATION: invalid input shape"); - } - - x = info.add_common_op("add", x, skip); - // Convert to float before reduce_mean - // Fp16 reduce_mean on GPU causes loss of accuracy - auto float_x = info.add_instruction( - make_op("convert", {{"target_type", migraphx::shape::float_type}}), x); - auto x_sq = info.add_common_op("mul", float_x, float_x); - auto rms = info.add_instruction(make_op("reduce_mean", {{"axes", {axis}}}), x_sq); - rms = info.add_instruction(make_op("convert", {{"target_type", x_dtype}}), rms); - auto mean = rms; - epsilon = - (x_dtype == migraphx::shape::half_type and std::abs(epsilon) < 1e-7) ? 1e-7 : epsilon; - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_dtype}, {epsilon}}); - rms = info.add_common_op("add", rms, eps); - auto rrms = info.add_instruction(make_op("rsqrt"), rms); - auto result = info.add_common_op("mul", x, rrms); - result = info.add_common_op("mul", result, gamma); - if(args.size() == 4) - { - result = info.add_common_op("add", result, bias); - x = info.add_common_op("add", x, bias); - } - - // Outputs (1 - 4) - // output : T - // 3D output tensor with shape (batch_size, sequence_length, hidden_size)Or 2D output tensor - // with shape (token_count, hidden_size) - // mean (optional) : U Saved mean used during training - // to speed up gradient computation - // inv_std_var (optional) : U Saved inverse standard - // variance used during training to speed up gradient computation. - // input_skip_bias_sum (optional) : T Sum of the input and skip inputs (and bias if it - // exists)with shape (batch_size, sequence_length, hidden_size) or (token_count, - // hidden_size). - - return {result, mean, rrms, x}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_slice.cpp b/docker/rocm/migraphx/onnx/parse_slice.cpp deleted file mode 100644 index 57e546694..000000000 --- a/docker/rocm/migraphx/onnx/parse_slice.cpp +++ /dev/null @@ -1,177 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_slice : op_parser -{ - - std::vector operators() const { return {{"Slice"}}; } - - struct slice_desc - { - op::slice op; - std::vector op_args; - std::vector steps; - std::vector raxes; - - void always_insert(instruction_ref arg) { op_args.insert(op_args.begin(), arg); } - - /** - * Either insert argument into `this->op_args` or return the constant value of the argument - */ - std::vector insert(instruction_ref arg) - { - std::vector result; - migraphx::argument arg_value = arg->eval(); - if(arg_value.empty()) - { - op_args.insert(op_args.begin(), arg); - } - else - { - arg_value.visit([&](auto s) { result.assign(s.begin(), s.end()); }); - } - return result; - } - }; - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - auto sd = construct_slice_desc(parser, info, args); - auto ins = info.add_instruction(sd.op, sd.op_args); - if(not sd.raxes.empty()) - { - ins = info.add_instruction(make_op("reverse", {{"axes", sd.raxes}}), ins); - } - // If any steps are other than default 1, add a "steps" op - if(std::any_of(sd.steps.begin(), sd.steps.end(), [](auto s) { return std::abs(s) != 1; })) - { - std::vector nsteps; - std::transform(sd.steps.begin(), - sd.steps.end(), - std::back_inserter(nsteps), - [](auto s) { return std::abs(s); }); - return ins = info.add_instruction( - make_op("step", {{"axes", sd.op.axes}, {"steps", nsteps}}), ins); - } - else - return ins; - } - - slice_desc construct_slice_desc(const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - slice_desc sd; - - // slice can have up to 5 inputs, we first check the 5th one - // to decide whether MIGRAPHX can handle this slice. - if(args.size() == 5) - { - migraphx::argument step_arg = args.back()->eval(); - check_arg_empty(step_arg, "PARSE_SLICE: cannot handle variable steps for slice"); - step_arg.visit([&](auto s) { sd.steps.assign(s.begin(), s.end()); }); - } - - if(args.size() >= 4) - { - sd.op.axes = sd.insert(args.at(3)); - } - else if(contains(info.attributes, "axes")) - { - literal s = parser.parse_value(info.attributes.at("axes")); - s.visit([&](auto v) { copy(v, std::back_inserter(sd.op.axes)); }); - } - - if(args.size() >= 3) - { - sd.op.ends = sd.insert(args.at(2)); - } - else if(contains(info.attributes, "ends")) - { - literal s = parser.parse_value(info.attributes.at("ends")); - s.visit([&](auto v) { copy(v, std::back_inserter(sd.op.ends)); }); - } - - if(args.size() >= 2) - { - sd.op.starts = sd.insert(args.at(1)); - } - else if(contains(info.attributes, "starts")) - { - literal s = parser.parse_value(info.attributes.at("starts")); - s.visit([&](auto v) { copy(v, std::back_inserter(sd.op.starts)); }); - } - - // data input argument - sd.always_insert(args.at(0)); - - // If axes arg is not given, the default is all of them. - if(sd.op.axes.empty() and sd.op_args.size() <= 3) - { - std::vector axes(args[0]->get_shape().ndim()); - std::iota(axes.begin(), axes.end(), int64_t{0}); - sd.op.axes = axes; - } - - if(std::any_of(sd.steps.begin(), sd.steps.end(), [](auto s) { return s != 1; })) - { - if(sd.op.starts.empty() or sd.op.ends.empty()) - MIGRAPHX_THROW( - "PARSE_SLICE: steps and variable starts and/or ends is not supported"); - if(sd.op.axes.empty()) - MIGRAPHX_THROW("PARSE_SLICE: steps and variable axes is not supported"); - } - - // If any axes have negative step, prepare to add a "reverse" op - for(auto i : range(sd.steps.size())) - { - if(sd.steps[i] >= 0) - continue; - sd.op.starts[i] += 1; - if(sd.op.starts[i] == 0) - sd.op.starts[i] = INT_MAX; - sd.op.ends[i] += 1; - sd.raxes.push_back(sd.op.axes[i]); - std::swap(sd.op.starts[i], sd.op.ends[i]); - } - return sd; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_softmax.cpp b/docker/rocm/migraphx/onnx/parse_softmax.cpp deleted file mode 100644 index 2f079c38d..000000000 --- a/docker/rocm/migraphx/onnx/parse_softmax.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_softmax : op_parser -{ - std::vector operators() const - { - return {{"Softmax", "softmax"}, {"LogSoftmax", "logsoftmax"}}; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - // default axis value is -1 for opset 13 - int64_t axis = -1; - - // axis value is 1 for previous opset versions - if(parser.opset_version < 13) - { - axis = 1; - } - - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - - return info.add_instruction(make_op(opd.op_name, {{"axis", axis}}), args); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_softmaxcrossentropyloss.cpp b/docker/rocm/migraphx/onnx/parse_softmaxcrossentropyloss.cpp deleted file mode 100644 index 581a8ed30..000000000 --- a/docker/rocm/migraphx/onnx/parse_softmaxcrossentropyloss.cpp +++ /dev/null @@ -1,491 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/* ------------------------ -SoftmaxCrossEntropyLoss ------------------------ -Loss function that measures the softmax cross entropy between 'scores' and 'labels'. -This operator first computes a loss tensor whose shape is identical to the labels input. -If the input is 2-D with shape (N, C), the loss tensor may be a N-element -vector L = (l_1, l_2, ..., l_N). If the input is N-D tensor with -shape (N, C, D1, D2, ..., Dk), the loss tensor L may have (N, D1, D2, ..., Dk) -as its shape and L[i,][j_1][j_2]...[j_k] denotes a scalar element in L. -After L is available, this operator can optionally do a reduction operator. - -shape(scores): (N, C) where C is the number of classes, or (N, C, D1, D2,..., Dk), - with K >= 1 in case of K-dimensional loss. -shape(labels): (N) where each value is 0 <= labels[i] <= C-1, or (N, D1, D2,..., Dk), - with K >= 1 in case of K-dimensional loss. - - -The loss for one sample, l_i, can calculated as follows: - -l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk], where i is the index of classes. -or - -l[i][d1][d2]...[dk] = -y[i][c][d1][d2]..[dk] * weights[c], if 'weights' is provided. -loss is zero for the case when label-value equals ignore_index. - -l[i][d1][d2]...[dk] = 0, when labels[n][d1][d2]...[dk] = ignore_index -where: - -p = Softmax(scores) -y = Log(p) -c = labels[i][d1][d2]...[dk] -Finally, L is optionally reduced: - -If reduction = 'none', the output is L with shape (N, D1, D2, ..., Dk). -If reduction = 'sum', the output is scalar: Sum(L). -If reduction = 'mean', the output is scalar: ReduceMean(L), or - if weight is provided: ReduceSum(L) / ReduceSum(W), - where tensor W is of shape (N, D1, D2, ..., Dk) and W[n][d1][d2]...[dk] = -weights[labels[i][d1][d2]...[dk]]. - -Attributes -+++++++++++ - -ignore_index : int - Specifies a target value that is ignored and - does not contribute to the input gradient. It's an optional value. - -reduction : string (default is mean) - Type of reduction to apply to loss: none, sum, mean(default). - - 'none': no reduction will be applied - - 'sum': the output will be summed. - - 'mean': the sum of the output will be divided by - the number of elements in the output. - -Inputs (2 - 3) -++++++++++++++ - -scores (differentiable) : T - The predicted outputs with shape [batch_size, class_size], or [batch_size, class_size, D1, D2 , -..., Dk], where K is the number of dimensions. - -labels (non-differentiable) : Tind - The ground truth output tensor, with shape [batch_size], or [batch_size, D1, D2, ..., Dk], - where K is the number of dimensions. Labels element value shall be in range of [0, C). - If ignore_index is specified, it may have a value outside [0, C) and the label values should - either be in the range [0, C) or have the value ignore_index. - -weights (optional, non-differentiable) : T - A manual rescaling weight given to each class. If given, - it has to be a 1D Tensor assigning weight to each of the classes. - Otherwise, it is treated as if having all ones. - -Outputs (1 - 2) -================== - -output (differentiable) : T - Weighted loss float Tensor. If reduction is 'none', this has the shape of [batch_size], - or [batch_size, D1, D2, ..., Dk] in case of K-dimensional loss. Otherwise, it is a scalar. - -log_prob (optional, differentiable) : T - Log probability tensor. If the output of softmax is prob, its value is log(prob). - -Type Constraints -=================== - T : tensor(float16), tensor(float), tensor(double), tensor(bfloat16) (Currently not supported in -MIGX) Constrain input and output types to float tensors. - -Tind : tensor(int32), tensor(int64) - Constrain target to integer types -*/ - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_softmaxcrossentropyloss : op_parser -{ - std::vector operators() const - { - return {{"SoftmaxCrossEntropyLoss", "softmaxcrossentropyloss"}, - {"NegativeLogLikelihoodLoss", "negativelikelihoodloss"}}; - } - - // Handle ignore index if it's within range of allowable classes - // return false if ignore index out of bounds and never add literal to graph - // return true if ignore index is in bound and pass back literal - bool normalize_input_index(const onnx_parser& parser, - const onnx_parser::node_info& info, - instruction_ref& ignore_index, - int64_t& ignore_index_val) const - { - bool has_ignore_index = contains(info.attributes, "ignore_index"); - if(has_ignore_index) - { - ignore_index_val = parser.parse_value(info.attributes.at("ignore_index")).at(); - - ignore_index = info.add_literal(migraphx::literal( - migraphx::shape(migraphx::shape::int64_type, {1}, {0}), {ignore_index_val})); - - return true; - } - return false; - } - - std::string get_reduction_param(const onnx_parser::node_info& info, - const std::string& name) const - { - std::string reduction = "mean"; - if(contains(info.attributes, "reduction")) - { - reduction = info.attributes.at("reduction").s(); - if(not contains({"mean", "sum", "none"}, reduction)) - { - MIGRAPHX_THROW(name + ":Invalid reduction mode: " + reduction + - "\n Valid options are [none, mean, sum]"); - } - } - return reduction; - } - - instruction_ref get_scores(const instruction_ref& arg, const std::string& name) const - { - auto scores = arg; - auto scores_shape = scores->get_shape(); - if(scores_shape.ndim() < 2) - { - MIGRAPHX_THROW(name + - "Scores must be two or more dimensions [batch, class_size, D1...Dk]"); - } - - if(migraphx::shape::is_integral(scores_shape.type())) - { - MIGRAPHX_THROW(name + ": Score must be either half, float, or double type"); - } - return scores; - } - - instruction_ref - get_labels(const instruction_ref& arg, const std::string& name, const shape& scores_shape) const - { - auto labels = arg; - auto label_shape = labels->get_shape(); - - if(label_shape.type() != migraphx::shape::int32_type and - label_shape.type() != migraphx::shape::int64_type) - { - MIGRAPHX_THROW(name + ": Labels must either be int32 or int64 types"); - } - - if(scores_shape.lens()[0] != label_shape.lens()[0]) - { - MIGRAPHX_THROW(name + ": Score and Labels must identical batch size inputs"); - } - - if((scores_shape.ndim() - 1) != label_shape.ndim()) - { - MIGRAPHX_THROW(name + ": Score and Labels must contain identical K-Dimensions"); - } - - // Check that K-Dimensions are equal between scores and labels - if(label_shape.ndim() > 1) - { - auto score_len = scores_shape.lens(); - auto label_len = label_shape.lens(); - - if(not std::equal(score_len.begin() + 2, score_len.end(), label_len.begin() + 1)) - { - MIGRAPHX_THROW(name + ": K-Dimensions must be equal values between " - "score and labels"); - } - } - - return labels; - } - - instruction_ref get_weights(const onnx_parser::node_info& info, - const std::string& name, - const std::vector& args, - const shape& scores_shape, - size_t class_size) const - { - // Default weights will always be 1 - auto weights = info.add_literal( - migraphx::literal(migraphx::shape(scores_shape.type(), {1}, {0}), {1})); - weights = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", {class_size}}}), weights); - - bool has_weights = (args.size() > 2); - // Get optional input weights (Used for mean reduction) - if(has_weights) - { - weights = args.at(2); - auto weights_shape = weights->get_shape(); - - if(weights_shape.lens()[0] != scores_shape.lens()[1]) - { - MIGRAPHX_THROW(name + ": Invalid weight vector shape. Weight must " - "contain weight for each class"); - } - - if(migraphx::shape::is_integral(weights_shape.type())) - { - MIGRAPHX_THROW(name + ": weight must be either half, float, or double type"); - } - - if(weights_shape.type() != scores_shape.type()) - { - MIGRAPHX_THROW(name + ": Weight and Scores inputs must be the same type"); - } - } - - // Always make weights negative saves pointwise after indexing - weights = info.add_instruction(migraphx::make_op("neg"), weights); - - return weights; - } - - instruction_ref handle_index_selection(const onnx_parser::node_info& info, - const instruction_ref labels) const - { - // Pick out the coordinates from the inputs to gerneate the proper indicies to gather - // what will be operated on later. - - // Use label indices to select weights - auto labels_unsq = - info.add_instruction(migraphx::make_op("unsqueeze", {{"axes", {-1}}}), labels); - auto label_shape = labels->get_shape(); - auto labels_rank = labels_unsq->get_shape().ndim(); - - std::vector coordinate_index_literals; - auto lengths = labels_unsq->get_shape().lens(); - - for(size_t axis = 0; axis < (labels_rank - 1); axis++) - { - auto len_val = lengths.at(axis); - // Trying to replicate torch arrange() here. - std::vector vect_of_lit(len_val); - std::iota(vect_of_lit.begin(), vect_of_lit.end(), 0); - auto batch_dim_indicies = - info.add_literal(migraphx::shape(label_shape.type(), {len_val}), vect_of_lit); - - // This is supposed to do unsq_dims = [:a] + [a + 1:] - std::vector unsq_dims(labels_rank); - std::iota(unsq_dims.begin(), unsq_dims.end(), 0); - auto it = unsq_dims.begin(); - it += axis; - unsq_dims.erase(it); - - auto batch_dim_index_unsq = info.add_instruction( - migraphx::make_op("unsqueeze", {{"axes", unsq_dims}}), batch_dim_indicies); - - auto batch_dim_indicies_bc = info.add_instruction( - migraphx::make_op("multibroadcast", - {{"out_lens", labels_unsq->get_shape().lens()}}), - batch_dim_index_unsq); - coordinate_index_literals.push_back(batch_dim_indicies_bc); - } - - coordinate_index_literals.push_back(labels_unsq); - return info.add_instruction(migraphx::make_op("concat", {{"axis", -1}}), - coordinate_index_literals); - } - - instruction_ref handle_reduction(const onnx_parser::node_info& info, - const instruction_ref loss_tensor, - const instruction_ref weights, - const std::string& reduction, - bool has_weights) const - { - instruction_ref final_loss_tensor = loss_tensor; - - // Used for reductions - std::vector loss_dims(loss_tensor->get_shape().ndim()); - std::iota(loss_dims.begin(), loss_dims.end(), 0); - - // Add reduction step after we're generated crossentropyloss tensor and rearragned weight - // scaling tensor - if(reduction == "mean" and has_weights) - { - std::vector weight_dims(weights->get_shape().ndim()); - std::iota(weight_dims.begin(), weight_dims.end(), 0); - - final_loss_tensor = info.add_instruction( - migraphx::make_op("reduce_sum", {{"axes", loss_dims}}), final_loss_tensor); - auto reduced_weights = info.add_instruction( - migraphx::make_op("reduce_sum", {{"axes", weight_dims}}), weights); - reduced_weights = info.add_instruction(migraphx::make_op("neg"), reduced_weights); - - final_loss_tensor = - info.add_instruction(migraphx::make_op("div"), final_loss_tensor, reduced_weights); - } - else if(reduction == "mean" and not has_weights) - { - final_loss_tensor = info.add_instruction( - migraphx::make_op("reduce_mean", {{"axes", loss_dims}}), final_loss_tensor); - } - else if(reduction == "sum") - { - final_loss_tensor = info.add_instruction( - migraphx::make_op("reduce_sum", {{"axes", loss_dims}}), final_loss_tensor); - } - - return final_loss_tensor; - } - - instruction_ref handle_ignored_labels(const onnx_parser::node_info& info, - const instruction_ref labels, - const instruction_ref ignore_index, - const instruction_ref loss_tensor) const - { - auto labels_shape = labels->get_shape(); - auto ignore_idx_bc = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", labels_shape.lens()}}), ignore_index); - auto conv_labels = info.add_instruction( - migraphx::make_op("convert", {{"target_type", ignore_index->get_shape().type()}}), - labels); - - std::vector zero_val_vect(labels_shape.elements(), 0); - auto zero_vector = - info.add_literal(migraphx::literal(loss_tensor->get_shape(), zero_val_vect)); - auto equals_mask = - info.add_instruction(migraphx::make_op("equal"), conv_labels, ignore_idx_bc); - - // If the any label is equal to ignore index, zero out the final tensor value - return info.add_instruction( - migraphx::make_op("where"), equals_mask, zero_vector, loss_tensor); - } - - std::vector parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - const std::vector& args) const - { - // Get the op name to be used for parsing - std::string op_name{opd.op_name}; - auto is_softmaxcrossentropy = (opd.op_name == "softmaxcrossentropyloss"); - - // Get and handle attributes - auto reduction = get_reduction_param(info, op_name); - - // Get and validate Inputs - auto scores = get_scores(args.at(0), op_name); - auto scores_shape = scores->get_shape(); - auto labels = get_labels(args.at(1), op_name, scores_shape); - - // Meta parameters based on input scores shape - size_t ndims = scores_shape.ndim(); - size_t class_size = scores_shape.lens().at(1); - bool is_k_dim = (ndims >= 3); - // Ensure first k-th dimension is greater then one if ndims == 3 - if(ndims == 3) - { - auto last_dim = scores_shape.lens().at(2); - if(last_dim < 2) - is_k_dim = false; - } - - // Ignore_index is optional attribute, assign this as a scalar literal input to the op - instruction_ref ignore_index; - int64_t ignore_index_val = -1; - auto has_ignore_index = normalize_input_index(parser, info, ignore_index, ignore_index_val); - - bool has_weights = (args.size() > 2); - instruction_ref weights = get_weights(info, op_name, args, scores_shape, class_size); - - // Adjust weights based on ignore index if its in bounds of [0, class_size) if that's set to - // reduce output after mul to zero. Saves us from doing a where() here and just scale at the - // end - if(has_ignore_index and (ignore_index_val < class_size and ignore_index_val >= 0)) - { - auto weights_shape = weights->get_shape(); - std::vector zero_val_vect(weights_shape.elements(), 0); - auto zero_val = info.add_literal(migraphx::literal(weights_shape, zero_val_vect)); - weights = info.add_instruction( - migraphx::make_op("scatter_none", {{"axis", 0}}), weights, ignore_index, zero_val); - } - - if(is_softmaxcrossentropy) - { - // Need to perform softmax on all the data before we select final axes - scores = info.add_instruction(migraphx::make_op("softmax", {{"axis", 1}}), scores); - } - - // Index selection before loss calculation completed - auto gathernd_indicies = handle_index_selection(info, labels); - - std::vector perm(class_size, 0); - if(is_k_dim) - { - std::iota(perm.begin() + 1, perm.end(), 2); - perm.at(class_size - 1) = 1; - scores = info.add_instruction(migraphx::make_op("transpose", {{"permutation", perm}}), - scores); - } - - scores = info.add_instruction(migraphx::make_op("gathernd"), scores, gathernd_indicies); - - std::vector axis_list(ndims - 1, 0); - std::iota((axis_list.begin() + 1), axis_list.end(), 2); - weights = - info.add_instruction(migraphx::make_op("unsqueeze", {{"axes", axis_list}}), weights); - weights = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", scores_shape.lens()}}), weights); - if(is_k_dim) - weights = info.add_instruction(migraphx::make_op("transpose", {{"permutation", perm}}), - weights); - weights = info.add_instruction(migraphx::make_op("gathernd"), weights, gathernd_indicies); - - // Do pointwise operators on the final set of indicies and scores we care about rather than - // before so that we're not doing a bunch of pointwise on items that aren't part of the loss - // calulation. - auto log_sm_scores = scores; - if(is_softmaxcrossentropy) - { - log_sm_scores = info.add_instruction(migraphx::make_op("log"), scores); - } - - // Always multiply out the weights. - auto weighted_result = - info.add_instruction(migraphx::make_op("mul"), log_sm_scores, weights); - - auto loss_tensor = handle_reduction(info, weighted_result, weights, reduction, has_weights); - - // Handle the case where label == ignore_index regardless if label or ignore index are - // outside of the range of [0, Class_size) - if(has_ignore_index and ((ignore_index_val < 0) or (ignore_index_val >= class_size))) - { - loss_tensor = handle_ignored_labels(info, labels, ignore_index, loss_tensor); - } - - if(is_softmaxcrossentropy) - return {loss_tensor, log_sm_scores}; - else - return {loss_tensor}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_softplus.cpp b/docker/rocm/migraphx/onnx/parse_softplus.cpp deleted file mode 100644 index 3b6c5dad6..000000000 --- a/docker/rocm/migraphx/onnx/parse_softplus.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_softplus : op_parser -{ - std::vector operators() const { return {{"Softplus"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - // Apply pointwise formula: y = ln(exp(x) + 1) - auto mb_ones = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", args[0]->get_shape().lens()}}), - info.add_literal(migraphx::literal{migraphx::shape{args[0]->get_shape().type()}, {1}})); - auto exp = info.add_instruction(migraphx::make_op("exp"), args[0]); - auto add = info.add_instruction(migraphx::make_op("add"), exp, mb_ones); - return info.add_instruction(migraphx::make_op("log"), add); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_softsign.cpp b/docker/rocm/migraphx/onnx/parse_softsign.cpp deleted file mode 100644 index b24f65746..000000000 --- a/docker/rocm/migraphx/onnx/parse_softsign.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_softsign : op_parser -{ - std::vector operators() const { return {{"Softsign"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - // Apply pointwise formula: y = x / (1 + |x|) - auto mb_ones = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", args[0]->get_shape().lens()}}), - info.add_literal(migraphx::literal{migraphx::shape{args[0]->get_shape().type()}, {1}})); - auto abs = info.add_instruction(migraphx::make_op("abs"), args[0]); - auto add = info.add_instruction(migraphx::make_op("add"), abs, mb_ones); - return info.add_instruction(migraphx::make_op("div"), args[0], add); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_spacetodepth.cpp b/docker/rocm/migraphx/onnx/parse_spacetodepth.cpp deleted file mode 100644 index d4b9e4621..000000000 --- a/docker/rocm/migraphx/onnx/parse_spacetodepth.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_spacetodepth : op_parser -{ - std::vector operators() const { return {{"SpaceToDepth"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - auto s = args[0]->get_shape(); - // blocksize attribute of SpaceToDepth - int blocksize = 1; // if blockSize of 1 then, this is a no-op - if(contains(info.attributes, "blocksize")) - { - blocksize = info.attributes.at("blocksize").i(); - } - if(blocksize < 1) - { - // blockSize less than 1 would rather result in DepthToSpace instead of SpaceToDepth - MIGRAPHX_THROW("SpaceToDepth: blocksize is less than 1"); - } - // calculate dimensions - auto res_lens = s.lens(); // {N, C, H, W} - if(((res_lens[2] % blocksize) == 0) and ((res_lens[3] % blocksize) == 0)) - { - // Co = C * (blocksize ^ 2) - res_lens[1] = res_lens[1] * blocksize * blocksize; - // Ho = (H / blocksize) - res_lens[2] = res_lens[2] / blocksize; - // Wo = (W / blocksize) - res_lens[3] = res_lens[3] / blocksize; - } // res_shape = (N, Co, Ho, Wo) - else - MIGRAPHX_THROW("SpaceToDepth: div by blocksize quotient not int "); - - auto trans_lens = s.lens(); // {N, C, H, W} - trans_lens[2] = res_lens[2]; - trans_lens[3] = blocksize; - trans_lens.push_back(res_lens[3]); - trans_lens.push_back(blocksize); // {N, C, Ho, blocksize, Wo, blocksize} - std::vector perm = {0, 3, 5, 1, 2, 4}; - auto temp1 = info.add_instruction(make_op("reshape", {{"dims", trans_lens}}), args[0]); - auto temp2 = info.add_instruction(make_op("transpose", {{"permutation", perm}}), temp1); - return info.add_instruction(make_op("reshape", {{"dims", res_lens}}), temp2); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_split.cpp b/docker/rocm/migraphx/onnx/parse_split.cpp deleted file mode 100644 index 355907b74..000000000 --- a/docker/rocm/migraphx/onnx/parse_split.cpp +++ /dev/null @@ -1,198 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -auto parse_dyn_split(const onnx_parser::node_info& info, - const std::vector& args, - int64_t tuned_axis) -{ - if(contains(info.attributes, "split")) - { - MIGRAPHX_THROW("PARSE_SPLIT: dynamic input and non-fixed split axis and `split` " - "attribute not supported"); - } - if(args.size() == 2) - { - MIGRAPHX_THROW("PARSE_SPLIT: dynamic input and non-fixed split axis and `split` " - "input not supported"); - } - - std::size_t num_outputs = info.num_outputs; - std::vector ret_ins(num_outputs); - - // Doing shape calculations for the splits in the graph - auto split_dim = info.add_instruction( - make_op("dimensions_of", {{"start", tuned_axis}, {"end", tuned_axis + 1}}), args[0]); - shape int64_scalar_shape{shape::int64_type, {1}, {0}}; - auto num_outputs_lit = info.add_literal(literal{int64_scalar_shape, {num_outputs}}); - auto num_outputs_minus_1_lit = info.add_literal(literal{int64_scalar_shape, {num_outputs - 1}}); - // (A + (B - 1)) / B == ceil(A / B) - auto chunk_size = info.add_instruction( - make_op("div"), - info.add_instruction(make_op("add"), split_dim, num_outputs_minus_1_lit), - num_outputs_lit); - for(int n = 0; n < num_outputs - 1; ++n) - { - // slice(input, starts = {n * chunk_size}, ends = {(n+1) * chunk_size}); axes = - // {tuned_axis} - ret_ins.at(n) = info.add_instruction( - make_op("slice", {{"axes", {tuned_axis}}}), - args[0], - info.add_instruction( - make_op("mul"), chunk_size, info.add_literal(literal{int64_scalar_shape, {n}})), - info.add_instruction(make_op("mul"), - chunk_size, - info.add_literal(literal{int64_scalar_shape, {n + 1}}))); - } - // last slice: slice(input, starts = {n * chunk_size}); ends = max_int, axes = - // {tuned_axis} - ret_ins.at(num_outputs - 1) = info.add_instruction( - make_op("slice", {{"axes", {tuned_axis}}, {"ends", {std::numeric_limits::max()}}}), - args[0], - info.add_instruction(make_op("mul"), - chunk_size, - info.add_literal(literal{int64_scalar_shape, {num_outputs - 1}}))); - return ret_ins; -} - -auto parse_static_split(const onnx_parser::node_info& info, - const onnx_parser& parser, - const std::vector& args, - int64_t tuned_axis) -{ - const auto& input_shape = args[0]->get_shape(); - // either static shape or fixed dynamic_dimension for split axis - auto tuned_axis_len = input_shape.to_static(0).lens().at(tuned_axis); - std::vector vec_splits; - if(contains(info.attributes, "split")) - { - literal s = parser.parse_value(info.attributes.at("split")); - s.visit([&](auto v) { vec_splits.assign(v.begin(), v.end()); }); - } - else if(args.size() == 2) - { - auto s = args[1]->eval(); - check_arg_empty(s, "PARSE_SPLIT: non-constant `split` input is not supported"); - s.visit([&](auto v) { vec_splits.assign(v.begin(), v.end()); }); - } - // no split attribute, input is equally divided - else - { - std::size_t num_outputs = info.num_outputs; - // the num_outputs attribute seems to be redundant since we already have - // node_info::num_outputs, but we can still perform an error check - if(contains(info.attributes, "num_outputs")) - { - num_outputs = parser.parse_value(info.attributes.at("num_outputs")).at(); - if(num_outputs != info.num_outputs) - { - MIGRAPHX_THROW("PARSE_SPLIT: num_outputs attribute " + std::to_string(num_outputs) + - " doesn't match actual number of outputs " + - std::to_string(info.num_outputs) + "!"); - } - } - if(tuned_axis_len % num_outputs == 0) - { - std::size_t chunk_size = tuned_axis_len / num_outputs; - vec_splits.resize(num_outputs, chunk_size); - } - else - { - std::size_t chunk_size = tuned_axis_len / num_outputs + 1; - std::size_t last_chunk_size = tuned_axis_len - chunk_size * (num_outputs - 1); - vec_splits.resize(num_outputs - 1, chunk_size); - vec_splits.push_back(last_chunk_size); - } - } - - if(std::accumulate(vec_splits.begin(), vec_splits.end(), int64_t(0)) != - static_cast(tuned_axis_len)) - { - MIGRAPHX_THROW( - "PARSE_SPLIT: sum of split attribute unequal to dim size of axis! tuned axis:" + - std::to_string(tuned_axis_len) + " Output " + to_string_range(vec_splits) + " Rank " + - std::to_string(input_shape.ndim())); - } - - std::vector ret_ins; - int64_t start = 0; - for(auto sl : vec_splits) - { - ret_ins.push_back(info.add_instruction( - make_op("slice", {{"axes", {tuned_axis}}, {"starts", {start}}, {"ends", {start + sl}}}), - args[0])); - start += sl; - } - - return ret_ins; -} - -struct parse_split : op_parser -{ - std::vector operators() const { return {{"Split"}}; } - - std::vector parse(const op_desc& opd, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - int64_t axis = 0; - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - - const auto& input_shape = args[0]->get_shape(); - // axis over which the split occurs (split_axis) - int64_t tuned_axis = tune_axis(input_shape.ndim(), axis, opd.onnx_name); - - auto split_axis_is_fixed = [&]() { - return input_shape.dyn_dims().at(tuned_axis).is_fixed(); - }; - - if(input_shape.dynamic() and not split_axis_is_fixed()) - { - return parse_dyn_split(info, args, tuned_axis); - } - else - { - return parse_static_split(info, parser, args, tuned_axis); - } - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_squeeze.cpp b/docker/rocm/migraphx/onnx/parse_squeeze.cpp deleted file mode 100644 index daf352f03..000000000 --- a/docker/rocm/migraphx/onnx/parse_squeeze.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_squeeze : op_parser -{ - std::vector operators() const - { - return {{"Squeeze", "squeeze"}, {"Unsqueeze", "unsqueeze"}}; - } - - operation assign_axes(operation& op, const std::vector& axes) const - { - auto v = op.to_value(); - v["axes"] = axes; - op.from_value(v); - - return op; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - auto op = parser.load(opd.op_name, info); - if(args.size() == 2) - { - auto arg_axes = args.at(1)->eval(); - check_arg_empty(arg_axes, "PARSE_" + opd.op_name + ": cannot handle variable axes!"); - std::vector axes; - arg_axes.visit([&](auto s) { axes.assign(s.begin(), s.end()); }); - op = assign_axes(op, axes); - } - - auto arg = info.make_contiguous(args.front()); - return info.add_instruction(op, arg); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_thresholdedrelu.cpp b/docker/rocm/migraphx/onnx/parse_thresholdedrelu.cpp deleted file mode 100644 index dbd7fb374..000000000 --- a/docker/rocm/migraphx/onnx/parse_thresholdedrelu.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_thresholdedrelu : op_parser -{ - std::vector operators() const { return {{"ThresholdedRelu"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - float alpha = 1.0; - if(contains(info.attributes, "alpha")) - alpha = parser.parse_value(info.attributes.at("alpha")).at(); - - auto x_shape = args[0]->get_shape(); - - auto lit_zero = info.add_literal(migraphx::literal{migraphx::shape{x_shape.type()}, {0}}); - auto lit_alpha = - info.add_literal(migraphx::literal{migraphx::shape{x_shape.type()}, {alpha}}); - auto mb_zero = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", x_shape.lens()}}), lit_zero); - auto mb_alpha = info.add_instruction( - migraphx::make_op("multibroadcast", {{"out_lens", x_shape.lens()}}), lit_alpha); - auto condition = info.add_instruction(migraphx::make_op("greater"), args[0], mb_alpha); - - return info.add_instruction(migraphx::make_op("where"), condition, args[0], mb_zero); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_tile.cpp b/docker/rocm/migraphx/onnx/parse_tile.cpp deleted file mode 100644 index ae1614b79..000000000 --- a/docker/rocm/migraphx/onnx/parse_tile.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_tile : op_parser -{ - std::vector operators() const { return {{"Tile"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - migraphx::argument arg_s = args[1]->eval(); - check_arg_empty(arg_s, "PARSE_TILE: dynamic shape is not supported"); - std::vector repeats; - arg_s.visit([&](auto input) { repeats.assign(input.begin(), input.end()); }); - - auto l0 = args[0]; - for(int i = 0; i < repeats.size(); i++) - { - auto l1 = l0; - for(int j = 1; j < repeats[i]; j++) - { - l0 = info.add_instruction(make_op("concat", {{"axis", i}}), l0, l1); - } - } - return l0; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_topk.cpp b/docker/rocm/migraphx/onnx/parse_topk.cpp deleted file mode 100644 index 66ab9f7ad..000000000 --- a/docker/rocm/migraphx/onnx/parse_topk.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_topk : op_parser -{ - std::vector operators() const { return {{"TopK"}}; } - - std::vector parse(const op_desc& /*opd*/, - const onnx_parser& parser, - onnx_parser::node_info info, - std::vector args) const - { - int64_t k = 0; - if(args.size() == 2) - { - auto arg_k = args.at(1)->eval(); - check_arg_empty(arg_k, "PARSE_TopK: k input must be constant"); - k = arg_k.at(); - } - else if(contains(info.attributes, "k")) - { - k = info.attributes.at("k").i(); - } - - bool largest = true; - if(contains(info.attributes, "largest")) - { - largest = static_cast(info.attributes.at("largest").i()); - } - - int64_t axis = -1; - if(contains(info.attributes, "axis")) - { - axis = parser.parse_value(info.attributes.at("axis")).at(); - } - - auto topk_ret = info.add_instruction( - make_op("topk", {{"k", k}, {"axis", axis}, {"largest", largest}}), args.at(0)); - - auto ret_val = info.add_instruction(make_op("get_tuple_elem", {{"index", 0}}), topk_ret); - auto ret_ind = info.add_instruction(make_op("get_tuple_elem", {{"index", 1}}), topk_ret); - - return {ret_val, ret_ind}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_transpose.cpp b/docker/rocm/migraphx/onnx/parse_transpose.cpp deleted file mode 100644 index 9799e73f2..000000000 --- a/docker/rocm/migraphx/onnx/parse_transpose.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_transpose : op_parser -{ - std::vector operators() const { return {{"Transpose"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - onnx_parser::node_info info, - std::vector args) const - { - std::vector perm{}; - if(contains(info.attributes, "perm")) - { - auto&& perm_vals = info.attributes["perm"].ints(); - perm = std::vector(perm_vals.begin(), perm_vals.end()); - } - - // if perm is empty, use the default value - auto n_dim = args.front()->get_shape().ndim(); - if(perm.empty()) - { - perm.resize(n_dim); - std::iota(perm.rbegin(), perm.rend(), 0); - } - - if(perm.size() != n_dim) - { - MIGRAPHX_THROW("PARSE_TRANSPOSE: perm and input have diffferent number of dims!"); - } - - return info.add_instruction(make_op("transpose", {{"permutation", perm}}), args.front()); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_trilu.cpp b/docker/rocm/migraphx/onnx/parse_trilu.cpp deleted file mode 100644 index 14aee908c..000000000 --- a/docker/rocm/migraphx/onnx/parse_trilu.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_trilu : op_parser -{ - std::vector operators() const { return {{"Trilu"}}; } - - instruction_ref parse(const op_desc&, - const onnx_parser&, - const onnx_parser::node_info& info, - std::vector args) const - { - auto input_shape = args[0]->get_shape(); - assert(input_shape.ndim() >= 2); - auto input_lens = input_shape.lens(); - - size_t num_rows = *(input_lens.rbegin() + 1); - size_t num_cols = input_lens.back(); - int k = 0; - bool upper = true; - - if(args.size() > 1) - { - auto arg_k = args[1]->eval(); - check_arg_empty(arg_k, "PARSE_TRILU: dynamic k not supported"); - k = arg_k.at(); - } - - if(contains(info.attributes, "upper")) - { - upper = static_cast(info.attributes.at("upper").i()); - } - - shape::type_t output_type = args[0]->get_shape().type(); - - // when creating the mask, if upper == 1, - // the inner triangle will have values set to 0 - std::vector mask_mat(num_rows * num_cols, upper); - // if upper == 0, kth diagonal must also be masked - if(not upper) - k++; - for(size_t i = 0; i < num_rows; i++) - { - for(int j = 0; j < std::min(k, static_cast(num_cols)); j++) - { - mask_mat[i * num_cols + j] = not upper; - } - k++; - } - - auto mask = info.add_literal( - migraphx::literal{migraphx::shape{output_type, {num_rows, num_cols}}, mask_mat}); - - return info.add_broadcastable_binary_op("mul", mask, args[0]); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_unique.cpp b/docker/rocm/migraphx/onnx/parse_unique.cpp deleted file mode 100644 index 873305ebe..000000000 --- a/docker/rocm/migraphx/onnx/parse_unique.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -// generate unique output stream y, given input stream x; -// -// case unsorted: -// input x: [2, 1, 1, 3, 4, 3], attr_sorted = 0; -// output(s): -// y: [2, 1, 3, 4] --- the unique output -// y_indices: [0, 1, 3, 4] --- first incidence, in terms of indices of x -// x_rev_indices: [0, 1, 1, 2, 3, 2] --- x seen in terms of indices of y -// y_count: [1, 2, 2, 1] -- count at each y_index. sum = len(x) -// -// case sorted: -// input x: [2, 1, 1, 3, 4, 3], attr_sorted = 1; -// output(s): -// y: [1, 2, 3, 4] --- the unique output -// y_indices: [1, 0, 3, 4] --- first incidence, in terms of indices of x -// x_rev_indices: [1, 0, 0, 2, 3, 2] --- x seen in terms of indices of y -// y_count: [2, 1, 2, 1] -- count at each y_index. sum = len(x) - -struct parse_unique : op_parser -{ - - std::vector operators() const { return {{"Unique"}}; } - - std::vector parse(const op_desc& opd, - const onnx_parser& parser, - const onnx_parser::node_info& info, - std::vector args) const - { - int64_t sorted = 1; // default = sorted. - - if(contains(info.attributes, "sorted")) - sorted = parser.parse_value(info.attributes.at("sorted")).at(); - - std::optional axis; - if(contains(info.attributes, "axis")) - { - auto n_dim = args[0]->get_shape().ndim(); - axis = parser.parse_value(info.attributes.at("axis")).at(); - axis = tune_axis(n_dim, *axis, opd.onnx_name); - } - migraphx::argument data_arg = args.back()->eval(); - - auto opr = axis ? migraphx::make_op("unique", {{"axis", *axis}, {"sorted", sorted}}) - : migraphx::make_op("unique", {{"sorted", sorted}}); - auto u_opr = info.add_instruction(opr, args.at(0)); - auto i_y = info.add_instruction(make_op("get_tuple_elem", {{"index", 0}}), u_opr); - auto i_y_idx = info.add_instruction(make_op("get_tuple_elem", {{"index", 1}}), u_opr); - auto i_x_idx = info.add_instruction(make_op("get_tuple_elem", {{"index", 2}}), u_opr); - auto i_count = info.add_instruction(make_op("get_tuple_elem", {{"index", 3}}), u_opr); - - return {i_y, i_y_idx, i_x_idx, i_count}; - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_variadic_op.cpp b/docker/rocm/migraphx/onnx/parse_variadic_op.cpp deleted file mode 100644 index 10b6cb60b..000000000 --- a/docker/rocm/migraphx/onnx/parse_variadic_op.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_variadic_op : op_parser -{ - std::vector operators() const - { - return {{"Sum", "add"}, {"Max", "max"}, {"Min", "min"}}; - } - - instruction_ref parse(const op_desc& opd, - const onnx_parser&, - onnx_parser::node_info info, - std::vector args) const - { - return std::accumulate(std::next(args.begin()), - args.end(), - args.front(), - [&](instruction_ref a, instruction_ref b) { - return info.add_broadcastable_binary_op(opd.op_name, a, b); - }); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/parse_where.cpp b/docker/rocm/migraphx/onnx/parse_where.cpp deleted file mode 100644 index bdbbfff5e..000000000 --- a/docker/rocm/migraphx/onnx/parse_where.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -struct parse_where : op_parser -{ - std::vector operators() const { return {{"Where"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const onnx_parser& /*parser*/, - const onnx_parser::node_info& info, - std::vector args) const - { - // TODO: broadcasting for dynamic shapes is only implemented - // for binary ops at time of writing, not ternary ops. - // When it becomes available, add multibroadcasting steps in the dynamic shape case. - // For now for dynamic shapes, just insert the Where op. All shapes must be the - // same for it to succeed. - if(std::all_of(args.begin(), args.end(), [](auto v) { return v->get_shape().dynamic(); })) - { - return info.add_instruction(make_op("where"), args[0], args[1], args[2]); - } - else if(std::none_of( - args.begin(), args.end(), [](auto v) { return v->get_shape().dynamic(); })) - { - // If shapes are static and any are broadcasted, insert multibroadcast ops - auto lens = - compute_broadcasted_lens(args[0]->get_shape().lens(), args[1]->get_shape().lens()); - lens = compute_broadcasted_lens(lens, args[2]->get_shape().lens()); - - if(args[0]->get_shape().lens() != lens) - { - args[0] = - info.add_instruction(make_op("multibroadcast", {{"out_lens", lens}}), args[0]); - } - - if(args[1]->get_shape().lens() != lens) - { - args[1] = - info.add_instruction(make_op("multibroadcast", {{"out_lens", lens}}), args[1]); - } - - if(args[2]->get_shape().lens() != lens) - { - args[2] = - info.add_instruction(make_op("multibroadcast", {{"out_lens", lens}}), args[2]); - } - - return info.add_instruction(make_op("where"), args[0], args[1], args[2]); - } - else - MIGRAPHX_THROW("PARSE_WHERE: doesn't support mixed static and dynamic shape inputs"); - } -}; - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/pooling.cpp b/docker/rocm/migraphx/onnx/pooling.cpp deleted file mode 100644 index 4eec85736..000000000 --- a/docker/rocm/migraphx/onnx/pooling.cpp +++ /dev/null @@ -1,247 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -value handle_pooling_values(const op_desc& opd, - onnx_parser::node_info info, - const shape& in_shape, - value values) -{ - auto kdims = in_shape.ndim() - 2; - if(starts_with(opd.onnx_name, "Global") or starts_with(opd.onnx_name, "QLinearGlobal")) - { - // if spatial dimensions are dynamic use dyn_global flag - if(in_shape.dynamic() and std::any_of(in_shape.dyn_dims().cbegin() + 2, - in_shape.dyn_dims().cend(), - [](auto dd) { return not dd.is_fixed(); })) - { - values["dyn_global"] = true; - values["lengths"] = std::vector(); - } - else - { - // works with static and fixed dynamic shape - auto m_lens = in_shape.max_lens(); - values["lengths"] = std::vector(m_lens.begin() + 2, m_lens.end()); - } - } - - if(contains(info.attributes, "ceil_mode")) - { - values["ceil_mode"] = static_cast(info.attributes.at("ceil_mode").i()); - } - - if(contains(info.attributes, "strides")) - { - values["stride"].clear(); - copy(info.attributes["strides"].ints(), std::back_inserter(values["stride"])); - check_attr_sizes(kdims, values["stride"].size(), "PARSE_POOLING: inconsistent strides"); - } - - if(contains(info.attributes, "kernel_shape")) - { - values["lengths"].clear(); - copy(info.attributes["kernel_shape"].ints(), std::back_inserter(values["lengths"])); - check_attr_sizes(kdims, values["lengths"].size(), "PARSE_POOLING: inconsistent lengths"); - } - - if(contains(info.attributes, "dilations")) - { - values["dilations"].clear(); - copy(info.attributes["dilations"].ints(), std::back_inserter(values["dilations"])); - check_attr_sizes( - kdims, values["dilations"].size(), "PARSE_POOLING: inconsistent dilations"); - } - - // lp_order attribute - if(contains(info.attributes, "p")) - { - values["lp_order"] = info.attributes.at("p").i(); - } - - // ensure pads available only when auto_pad is "NOT_SET" - check_padding_mode(info, opd.onnx_name); - - return values; -} - -instruction_ref add_pooling_op(const op_desc& opd, onnx_parser::node_info info, instruction_ref l0) -{ - std::string mode = opd.op_name; - const std::unordered_map mode_map = { - {"max", op::pooling_mode::max}, - {"average", op::pooling_mode::average}, - {"lpnorm", op::pooling_mode::lpnorm}}; - if(not contains(mode_map, mode)) - { - MIGRAPHX_THROW( - "PARSE_POOLING: onnx pooling mode must be [\"max\", \"average\", \"lpnorm\"]"); - } - operation op = make_op("pooling", {{"mode", mode_map.at(mode)}}); - value values = op.to_value(); - auto in_shape = l0->get_shape(); - assert(in_shape.ndim() > 2); - auto kdims = in_shape.ndim() - 2; - - values = handle_pooling_values(opd, info, in_shape, values); - - // count include padding, if count include pad is 1, we always use - // explicit pad - int count_include_pad = 0; - if(contains(info.attributes, "count_include_pad")) - { - if(in_shape.dynamic()) - { - MIGRAPHX_THROW("PARSE_POOLING: count_include_pad attribute is not supported for " - "dynamic input shape"); - } - count_include_pad = info.attributes.at("count_include_pad").i(); - } - - std::vector paddings; - float pad_val = ((mode == "max") ? std::numeric_limits::lowest() : 0.0f); - - if(contains(info.attributes, "pads")) - { - values["padding"].clear(); - copy(info.attributes["pads"].ints(), std::back_inserter(paddings)); - check_attr_sizes( - kdims, paddings.size() / 2, "PARSE_POOLING: inconsistent explicit paddings"); - } - - if(paddings.size() != 2 * kdims) - { - paddings.resize(kdims * 2); - std::fill_n(paddings.begin(), 2 * kdims, 0); - } - - if(values["padding"].size() != kdims) - { - values["padding"].resize(kdims); - std::fill_n(values["padding"].begin(), kdims, 0); - } - - if(values["stride"].size() != kdims) - { - values["stride"].resize(kdims); - std::fill_n(values["stride"].begin(), kdims, 1); - } - - if(values["dilations"].size() != kdims) - { - values["dilations"].resize(kdims); - std::fill_n(values["dilations"].begin(), kdims, 1); - } - - // used to calculate the supposed output shape - std::vector orig_padding = paddings; - - // TODO: add parsing for dilations - if(contains(info.attributes, "auto_pad") and - to_upper(info.attributes["auto_pad"].s()) != "NOTSET") - { - auto auto_pad = to_upper(info.attributes["auto_pad"].s()); - // don't use the given padding sizes, if any - // values["padding"].clear(); - if(in_shape.dynamic()) - { - // set padding_mode to trigger auto padding at runtime - bool is_same_upper = (auto_pad.find("SAME_UPPER") != std::string::npos); - values["padding_mode"] = is_same_upper ? to_value(op::padding_mode_t::same_upper) - : to_value(op::padding_mode_t::same_lower); - } - else - { - // Calculate auto padding - // dilations (argument 4) not supported; default to all 1's - cal_auto_padding_size(info, - values, - values["lengths"].to_vector(), - values["dilations"].to_vector(), - in_shape.lens(), - paddings); - values["padding"] = paddings; - // default padding_mode indicates that padding sizes are not calculated dynamically - values["padding_mode"] = migraphx::op::padding_mode_t::default_; - } - } - - std::vector slice_start; - std::vector slice_end; - tune_padding_size(values, paddings, count_include_pad, slice_start); - - if(not slice_start.empty()) - { - if(in_shape.dynamic()) - { - MIGRAPHX_THROW( - "PARSE_POOLING: asymmetric padding not supported for dynamic input shape"); - } - // calculate expected output shape - orig_padding.insert(orig_padding.begin() + kdims, 2, 0); - orig_padding.insert(orig_padding.begin(), 2, 0); - op::pad pad{orig_padding, 0.0f}; - shape padded_shape = pad.compute_shape({l0->get_shape()}); - - // make an op just to get its output shape - auto out_lens = make_op("pooling", values).compute_shape({padded_shape}).lens(); - // compute slice_end information - slice_end.resize(slice_start.size()); - std::transform(out_lens.begin() + 2, - out_lens.end(), - slice_start.begin(), - slice_end.begin(), - [](auto i, auto j) { return i + j; }); - } - values["padding"] = std::vector(paddings.begin(), paddings.end()); - - check_asym_padding(info, l0, paddings, values, count_include_pad, pad_val); - op.from_value(values); - - auto l1 = info.add_instruction(op, l0); - if(not slice_start.empty()) - { - std::vector axes(kdims); - std::iota(axes.begin(), axes.end(), 2); - l1 = info.add_instruction( - make_op("slice", {{"axes", axes}, {"starts", slice_start}, {"ends", slice_end}}), l1); - } - - return l1; -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/onnx/quantize_dequantize_linear.cpp b/docker/rocm/migraphx/onnx/quantize_dequantize_linear.cpp deleted file mode 100644 index f7ac2b0f1..000000000 --- a/docker/rocm/migraphx/onnx/quantize_dequantize_linear.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace onnx { - -std::vector -transform_quantize_dequantize_linear_inputs(const onnx_parser::node_info& info, - const std::string& onnx_name, - int block_size, - int axis, - std::vector args) -{ - const auto x = args.at(0); - const auto x_lens = x->get_shape().lens(); - const auto x_rank = x_lens.size(); - - instruction_ref y_scale = args.at(1); - const auto y_scale_lens = y_scale->get_shape().lens(); - const auto y_scale_rank = y_scale_lens.size(); - - // Per-tensor (per-layer) granularity - if(y_scale->get_shape().elements() == 1) - { - std::transform(args.begin() + 1, args.end(), args.begin() + 1, [&](auto ins) { - return info.add_instruction(make_op("multibroadcast", {{"out_lens", x_lens}}), ins); - }); - } - // Per-axis granularity - else if(y_scale_rank == 1) - { - axis = tune_axis(x_rank, axis, onnx_name); - if(x_lens[axis] != y_scale_lens[0]) - { - MIGRAPHX_THROW(onnx_name + - ": For per axis granularity the length of y_scale (actual: " + - to_string(y_scale_lens[0]) + ") must be equal to size of x on axis " + - to_string(axis) + "(actual: " + to_string(x_lens[axis]) + ")"); - } - - std::transform(args.begin() + 1, args.end(), args.begin() + 1, [&](auto ins) { - return info.add_instruction( - make_op("broadcast", {{"axis", axis}, {"out_lens", x_lens}}), ins); - }); - } - // Blocked granularity - else - { - axis = tune_axis(x_rank, axis, onnx_name); - - if(x_rank != y_scale_rank) - { - MIGRAPHX_THROW(onnx_name + ": x(rank: " + to_string(x_rank) + - ") and y_scale(rank: " + to_string(y_scale_rank) + - ") must be of same rank for block granularity"); - } - - for(auto i = 0u; i < x_lens.size(); ++i) - { - if(x_lens[i] != y_scale_lens[i] and i != axis) - { - MIGRAPHX_THROW(onnx_name + ": x(shape: " + to_string_range(x_lens) + - ") and y_scale(shape: " + to_string_range(y_scale_lens) + - ") shapes may only differ along provided axis(" + to_string(axis) + - ")"); - } - } - - // Given x shape (D0, ..., Di, ..., Dn), y_scale shape (S0, ... Si, ...Sn) and - // axis=i, the accepted range is [ceil(Di/Si), ceil(Di/(Si-1))-1] - float di = x_lens[axis]; - float si = y_scale_lens[axis]; - int block_size_min = std::ceil(di / si); - int block_size_max = std::ceil(di / (si - 1)) - 1; - // default block_size if not given is calculated (to support quark generated models): - if(block_size == 0) - block_size = block_size_min; - if(block_size < block_size_min or block_size > block_size_max) - MIGRAPHX_THROW(onnx_name + ": Block size(actual: " + to_string(block_size) + - ") must be within range [" + to_string(block_size_min) + ", " + - to_string(block_size_max) + "]"); - - std::transform(args.begin() + 1, args.end(), args.begin() + 1, [&](auto ins) { - if(block_size == 1) - return ins; - - ins = info.add_instruction(make_op("unsqueeze", {{"axes", {axis + 1}}}), ins); - - auto bc_lens = ins->get_shape().lens(); - bc_lens[axis + 1] = block_size; - ins = info.add_instruction(make_op("multibroadcast", {{"out_lens", bc_lens}}), ins); - - auto reshape_lens = x_lens; - reshape_lens[axis] = ins->get_shape().lens()[axis] * block_size; - ins = info.add_instruction(make_op("reshape", {{"dims", reshape_lens}}), ins); - - // Detect runt block - if(x_lens[axis] < reshape_lens[axis]) - { - ins = info.add_instruction( - make_op("slice", {{"axes", {axis}}, {"starts", {0}}, {"ends", {x_lens[axis]}}}), - ins); - } - - return ins; - }); - } - - return args; -} - -} // namespace onnx -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/op_enums.cpp b/docker/rocm/migraphx/op_enums.cpp deleted file mode 100644 index f8a967d1b..000000000 --- a/docker/rocm/migraphx/op_enums.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -// -// Supporting functions for enum values used in operator parameters. -// These values are declared as "enum class" and should include << streaming operators -// to be able to write their values in human-readable format so users can -// save and edit model files. -// -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace op { - -std::ostream& operator<<(std::ostream& os, pooling_mode v) -{ - // the strings for the enum are the same as the values used for onnx parsing - // but this enum is not onnx-specific: strings must be converted when parsing tf - static const std::vector pooling_mode_str = {"average", "max", "lpnorm"}; - os << pooling_mode_str[static_cast::type>(v)]; - return os; -} -std::ostream& operator<<(std::ostream& os, rnn_direction v) -{ - static const std::vector rnn_direction_str = { - "forward", "reverse", "bidirectional"}; - os << rnn_direction_str[static_cast::type>(v)]; - return os; -} - -} // namespace op -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/operation.cpp b/docker/rocm/migraphx/operation.cpp deleted file mode 100644 index e8b112fab..000000000 --- a/docker/rocm/migraphx/operation.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void migraphx_to_value(value& v, const operation& op) -{ - v["name"] = op.name(); - v["operator"] = op.to_value(); -} -void migraphx_from_value(const value& v, operation& op) -{ - op = make_op(v.at("name").to(), v.at("operator")); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/optimize_module.cpp b/docker/rocm/migraphx/optimize_module.cpp deleted file mode 100644 index 70853a147..000000000 --- a/docker/rocm/migraphx/optimize_module.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void optimize_module::apply(module_pass_manager& mpm) const -{ - mpm.get_module().repeat_while_changes(2, [&] { - // loop to further optimize after initial transformations - mpm.get_module().repeat_while_changes(4, [&] { - mpm.run_pass(simplify_reshapes{}); - mpm.run_pass(eliminate_convert{}); - mpm.run_pass(dead_code_elimination{}); - mpm.run_pass(simplify_algebra{}); - }); - mpm.run_pass(eliminate_common_subexpression{}); - mpm.run_pass(dead_code_elimination{}); - mpm.run_pass(propagate_constant{propagate_constant_skip_ops}); - mpm.run_pass(dead_code_elimination{}); - }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/pad_calc.cpp b/docker/rocm/migraphx/pad_calc.cpp deleted file mode 100644 index 3fe9603aa..000000000 --- a/docker/rocm/migraphx/pad_calc.cpp +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void calculate_padding(int64_t idx, - std::vector& pads, - int64_t input_dim, - int64_t stride, - int64_t dilation, - int64_t weight_dim, - bool is_same_upper) -{ - int64_t output_dim = (input_dim + stride - 1) / stride; // round up result - int64_t new_weight_dim = weight_dim + (weight_dim - 1) * (dilation - 1); - int64_t pad = - std::max(static_cast(0), (output_dim - 1) * stride + new_weight_dim - input_dim); - auto pad_ndims = pads.size() / 2; - - if(is_same_upper) - { - pads[idx] = pad / 2; - pads[idx + pad_ndims] = pad - pad / 2; - } - else - { - pads[idx + pad_ndims] = pad / 2; - pads[idx] = pad - pad / 2; - } -} - -/** - * Given the input array dimensions; kernel (wei_lens); strides; and dilations, - * calculate the padding value in each dimension. - * - */ -std::vector calc_dyn_auto_pad(const std::vector& input_lens, - const std::vector& wei_lens, - const std::vector& strides, - const std::vector& dilations, - bool use_upper) -{ - std::vector padding; - assert(input_lens.size() >= 3); - assert(input_lens.size() == wei_lens.size()); - std::size_t num_spatial_dims = input_lens.size() - 2; - padding.resize(2 * num_spatial_dims); - for(std::size_t i = 0; i < num_spatial_dims; i++) - { - std::ptrdiff_t input_dim = input_lens[i + 2]; - std::ptrdiff_t stride = strides[i]; - std::ptrdiff_t weight_dim = wei_lens[i + 2]; - std::ptrdiff_t dilation = dilations[i]; - std::ptrdiff_t output_dim = (input_dim + stride - 1) / stride; // round up result - std::ptrdiff_t new_weight_dim = weight_dim + (weight_dim - 1) * (dilation - 1); - std::size_t pad = std::max(static_cast(0), - (output_dim - 1) * stride + new_weight_dim - input_dim); - auto pad_ndims = padding.size() / 2; - - if(use_upper) - { - padding[i] = pad / 2; - padding[i + pad_ndims] = pad - pad / 2; - } - else - { - padding[i + pad_ndims] = pad / 2; - padding[i] = pad - pad / 2; - } - } - return padding; -} - -/** - * Calculate the correct output shape for a convolution with - * a given input size and other parameters. - * - */ -shape compute_padded_shape(const shape& input, - const shape& weights, - const std::vector& padding, - const std::vector& stride, - const std::vector& dilation) -{ - const size_t num_spatial_dims = input.lens().size() - 2; - - std::vector output_lens{input.lens()[0], weights.lens()[0]}; - // calculate the output shape of the convolution: ((W - K + 2P) / S) + 1 - for(size_t i = 0; i < num_spatial_dims; ++i) - { - auto padding_factor = padding[i] + padding[i + num_spatial_dims]; - output_lens.push_back(std::size_t(std::max( - 1, - (input.lens()[i + 2] - (1 + dilation[i] * (weights.lens()[i + 2] - 1)) + - padding_factor) / - stride[i] + - 1))); - } - return input.with_lens(output_lens); -} - -/** - * Calculate the correct output shape for a pooling with - * a given input size and other parameters. This uses - * the same formula for pooling that compute_padded_shape() uses - * for convolutions, but takes slightly different inputs. - * - */ -shape compute_padded_pool_shape(const shape& input, - const shape& kernel, - const std::vector& padding, - const std::vector& stride, - const std::vector& dilation) -{ - const size_t num_spatial_dims = input.lens().size() - 2; - - std::vector output_lens{input.lens()[0], input.lens()[1]}; - // calculate the output shape of the pooling: ((W - K + 2P) / S) + 1 - for(size_t i = 0; i < num_spatial_dims; ++i) - { - auto padding_factor = padding[i] + padding[i + num_spatial_dims]; - output_lens.push_back(std::size_t(std::max( - 1, - (input.lens()[i + 2] - (1 + dilation[i] * (kernel.lens()[i] - 1)) + padding_factor) / - stride[i] + - 1))); - } - return input.with_lens(output_lens); -} -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/param_utils.cpp b/docker/rocm/migraphx/param_utils.cpp deleted file mode 100644 index 9e985b7af..000000000 --- a/docker/rocm/migraphx/param_utils.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -std::string param_name(std::size_t i, const std::string& prefix) -{ - if(i < 10) - return prefix + std::to_string(i); - const std::size_t max_digits = 5; - if(i >= std::pow(10, max_digits)) - MIGRAPHX_THROW("Too many parameters."); - std::size_t n = log10(i) + 1; - return prefix + ":" + std::string(max_digits - n, '0') + std::to_string(i); -} - -void sort_params(std::vector& params) -{ - std::sort(params.begin(), params.end(), by(std::less<>{}, [](instruction_ref ins) { - const auto& param = any_cast(ins->get_operator()); - return param.parameter; - })); -} - -std::vector -find_inputs(const std::unordered_map& map_ins, - const_module_ref parent, - const_module_ref sub) -{ - std::vector result; - std::map names; - for(auto&& [input, param] : map_ins) - { - if(sub != nullptr and not sub->has_instruction(param)) - continue; - if(param->name() != "@param") - continue; - if(parent != nullptr and not parent->has_instruction(input)) - continue; - auto v = param->get_operator().to_value(); - auto name = v.at("parameter").to(); - names[name] = input; - } - std::transform(names.begin(), names.end(), std::back_inserter(result), [](const auto& p) { - return p.second; - }); - assert(not sub or result.size() == sub->get_parameter_shapes().size()); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/pass.cpp b/docker/rocm/migraphx/pass.cpp deleted file mode 100644 index ee8876678..000000000 --- a/docker/rocm/migraphx/pass.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/// Dummy pass for default return -struct id_pass -{ - std::string name() const { return "id"; } - void apply(const module&) const {} -}; - -pass enable_pass(bool enabled, pass p) -{ - if(enabled) - return p; - return id_pass{}; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/pass_manager.cpp b/docker/rocm/migraphx/pass_manager.cpp deleted file mode 100644 index f950277bd..000000000 --- a/docker/rocm/migraphx/pass_manager.cpp +++ /dev/null @@ -1,209 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_PASSES); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TIME_PASSES); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_PASSES); - -static bool is_pass_disabled(const std::string& name) -{ - static const auto passes = split_string(string_value_of(MIGRAPHX_DISABLE_PASSES{}, ""), ','); - return contains(passes, name); -} - -void validate_pass(module& mod, const pass& p, tracer trace) -{ - (void)mod; - (void)p; - (void)trace; -#ifndef NDEBUG - trace("Validate ..."); - auto invalid = mod.validate(); - if(invalid != mod.end()) - { - auto index = std::distance(mod.begin(), invalid); - MIGRAPHX_THROW(p.name() + " pass produces invalid program at instruction " + - std::to_string(index) + ": " + invalid->name()); - } - trace(); -#endif -} -void run_pass(program& prog, const pass& p, tracer trace) -{ - trace("Pass: ", p.name()); - p.apply(prog); - trace(prog); -} - -struct module_pm : module_pass_manager -{ - module* mod = nullptr; - module* root_mod = nullptr; - tracer* t = nullptr; - module* common_parent = nullptr; - program* prog = nullptr; - - module_pm(module* pmod = nullptr, tracer* pt = nullptr) : mod(pmod), t(pt) {} - - module_pm(module* pmod = nullptr, module* rmod = nullptr, tracer* pt = nullptr) - : mod(pmod), root_mod(rmod), t(pt) - { - } - - template - void trace(Ts&&... xs) const - { - assert(t); - (*t)(xs...); - } - - virtual module& get_module() override - { - assert(mod); - return *mod; - } - - virtual module* create_module(const std::string& name) override - { - assert(prog); - return prog->create_module(name); - } - - virtual module* create_module(const std::string& name, module m) override - { - assert(prog); - return prog->create_module(name, std::move(m)); - } - - virtual void rename_module(const std::string& old_name, const std::string& new_name) override - { - assert(prog); - assert(mod); - assert( - any_of(mod->get_sub_modules(), [&](module_ref sm) { return sm->name() == old_name; })); - prog->rename_module(old_name, new_name); - } - - virtual module* get_common_parent() override { return common_parent; } - - virtual module* get_root_module() override - { - if(root_mod != nullptr) - return root_mod; - assert(prog); - return prog->get_main_module(); - } - - virtual void run_pass(const pass& p) override - { - if(is_pass_disabled(p.name())) - return; - trace("Pass: ", p.name()); - assert(mod); - assert(mod->validate() == mod->end()); - if(enabled(MIGRAPHX_TIME_PASSES{})) - { - using milliseconds = std::chrono::duration; - auto ms = time([&] { p.apply(*this); }); - std::cout << p.name() << ": " << ms << "ms\n"; - } - else - { - p.apply(*this); - } - trace(*mod); - validate_pass(*mod, p, *t); - } -}; - -module& get_module(module_pass_manager& mpm) { return mpm.get_module(); } - -void run_passes(program& prog, module_ref root_mod, const std::vector& passes, tracer trace) -{ - if(enabled(MIGRAPHX_TRACE_PASSES{})) - trace = tracer{std::cout}; - std::unordered_set visited; - for(const auto& p : passes) - { - auto tree = prog.get_module_tree(); - std::vector sub_mods = root_mod->get_sub_modules(); - sub_mods.insert(sub_mods.begin(), root_mod); - visited.clear(); - for(const auto& mod : reverse(sub_mods)) - { - if(mod->bypass()) - continue; - if(not visited.insert(mod).second) - continue; - module_pm mpm{mod, root_mod, &trace}; - mpm.prog = &prog; - auto parents = range(tree.equal_range(mod)); - auto nparents = distance(parents); - if(nparents == 0) - mpm.common_parent = nullptr; - else if(nparents == 1) - mpm.common_parent = parents.begin()->second; - else - // Just set common parent to main module when there is muliple parents for now - // TODO: Compute the common parent - mpm.common_parent = prog.get_main_module(); - mpm.run_pass(p); - } - run_pass(prog, p, trace); - } -} - -void run_passes(module& mod, const std::vector& passes, tracer trace) -{ - if(enabled(MIGRAPHX_TRACE_PASSES{})) - trace = tracer{std::cout}; - for(const auto& p : passes) - { - module_pm{&mod, &mod, &trace}.run_pass(p); - } -} - -void run_passes(program& prog, const std::vector& passes, tracer trace) -{ - run_passes(prog, prog.get_main_module(), passes, trace); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/permutation.cpp b/docker/rocm/migraphx/permutation.cpp deleted file mode 100644 index f94997fc0..000000000 --- a/docker/rocm/migraphx/permutation.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -shape reorder_shape(const shape& s, const std::vector& permutation) -{ - return {s.type(), reorder_dims(s.lens(), permutation), reorder_dims(s.strides(), permutation)}; -} - -std::vector invert_permutation(const std::vector& permutation) -{ - return sort_permutation(permutation, std::less<>{}); -} - -std::vector find_permutation(const shape& s) -{ - std::vector result(s.lens().size()); - std::iota(result.begin(), result.end(), 0); - std::stable_sort(result.begin(), result.end(), by(std::greater<>{}, [&](auto x) { - return std::make_tuple(s.strides()[x], s.lens()[x]); - })); - return result; -} - -std::vector find_permutation(const std::vector& shapes) -{ - if(shapes.empty()) - return {}; - std::map, std::size_t> count; - for(auto&& s : shapes) - { - if(s.broadcasted()) - continue; - count[find_permutation(s)]++; - } - if(count.empty()) - { - std::vector r(shapes.front().lens().size()); - std::iota(r.begin(), r.end(), 0); - return r; - } - auto it = std::max_element( - count.begin(), count.end(), by(std::less<>{}, [](auto&& p) { return p.second; })); - assert(it != count.end()); - return it->first; -} - -std::vector normalize_permutation(const std::vector& shapes) -{ - auto result = shapes; - auto perm = find_permutation(shapes); - std::transform(result.begin(), result.end(), result.begin(), [&](auto s) { - return reorder_shape(s, perm); - }); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/preallocate_param.cpp b/docker/rocm/migraphx/preallocate_param.cpp deleted file mode 100644 index f6cd71837..000000000 --- a/docker/rocm/migraphx/preallocate_param.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void preallocate_param::apply(module& m) const -{ - auto last = std::prev(m.end()); - for(auto ins : iterator_for(m)) - { - if(ins->name() != "@param") - continue; - if(param != any_cast(ins->get_operator()).parameter) - continue; - std::string id = m.name() + ":" + param; - auto r = m.insert_instruction(ins, model.preallocate(ins->get_shape(), id)); - m.replace_instruction(ins, r); - m.move_instruction(ins, m.end()); - } - m.remove_instructions(std::next(last), m.end()); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/process.cpp b/docker/rocm/migraphx/process.cpp deleted file mode 100644 index 698a9959c..000000000 --- a/docker/rocm/migraphx/process.cpp +++ /dev/null @@ -1,461 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -// cppcheck-suppress definePrefix -#define WIN32_LEAN_AND_MEAN -#include -#include -#include -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_CMD_EXECUTE) - -#ifndef _WIN32 - -std::function redirect_to(std::ostream& os) -{ - return [&](const char* x) { os << x; }; -} - -template -int exec(const std::string& cmd, const char* type, F f) -{ - int ec = 0; - if(enabled(MIGRAPHX_TRACE_CMD_EXECUTE{})) - std::cout << cmd << std::endl; - auto closer = [&](FILE* stream) { - auto status = pclose(stream); - ec = WIFEXITED(status) ? WEXITSTATUS(status) : 0; // NOLINT - }; - { - // TODO: Use execve instead of popen - std::unique_ptr pipe(popen(cmd.c_str(), type), closer); // NOLINT - if(not pipe) - MIGRAPHX_THROW("popen() failed: " + cmd); - f(pipe.get()); - } - return ec; -} - -int exec(const std::string& cmd, const std::function& std_out) -{ - return exec(cmd, "r", [&](FILE* f) { - std::array buffer; - while(fgets(buffer.data(), buffer.size(), f) != nullptr) - std_out(buffer.data()); - }); -} - -int exec(const std::string& cmd, std::function std_in) -{ - return exec(cmd, "w", [&](FILE* f) { - std_in([&](const char* buffer, std::size_t n) { std::fwrite(buffer, 1, n, f); }); - }); -} - -#else - -constexpr std::size_t MIGRAPHX_PROCESS_BUFSIZE = 4096; - -enum class direction -{ - input, - output -}; - -template -class pipe -{ - public: - explicit pipe() - { - SECURITY_ATTRIBUTES attrs; - attrs.nLength = sizeof(SECURITY_ATTRIBUTES); - attrs.bInheritHandle = TRUE; - attrs.lpSecurityDescriptor = nullptr; - - if(CreatePipe(&m_read, &m_write, &attrs, 0) == FALSE) - throw GetLastError(); - - if(dir == direction::output) - { - // Do not inherit the read handle for the output pipe - if(SetHandleInformation(m_read, HANDLE_FLAG_INHERIT, 0) == 0) - throw GetLastError(); - } - else - { - // Do not inherit the write handle for the input pipe - if(SetHandleInformation(m_write, HANDLE_FLAG_INHERIT, 0) == 0) - throw GetLastError(); - } - } - - pipe(const pipe&) = delete; - pipe& operator=(const pipe&) = delete; - - pipe(pipe&&) = default; - - ~pipe() - { - if(m_write != nullptr) - { - CloseHandle(m_write); - } - if(m_read != nullptr) - { - CloseHandle(m_read); - } - } - - bool close_write_handle() - { - auto result = true; - if(m_write != nullptr) - { - result = CloseHandle(m_write) == TRUE; - m_write = nullptr; - } - return result; - } - - bool close_read_handle() - { - auto result = true; - if(m_read != nullptr) - { - result = CloseHandle(m_read) == TRUE; - m_read = nullptr; - } - return result; - } - - std::pair read(LPVOID buffer, DWORD length) const - { - DWORD bytes_read; - if(ReadFile(m_read, buffer, length, &bytes_read, nullptr) == FALSE and - GetLastError() == ERROR_MORE_DATA) - { - return {true, bytes_read}; - } - return {false, bytes_read}; - } - - HANDLE get_read_handle() const { return m_read; } - - bool write(LPCVOID buffer, DWORD length) const - { - DWORD bytes_written; - return WriteFile(m_write, buffer, length, &bytes_written, nullptr) == TRUE; - } - - HANDLE get_write_handle() const { return m_write; } - - private: - HANDLE m_write = nullptr, m_read = nullptr; -}; - -// clang-format off -template -int exec(const std::string& cmd, const std::string& cwd, const std::string& args, - const std::string& envs, F f) -// clang-format on -{ - if(enabled(MIGRAPHX_TRACE_CMD_EXECUTE{})) - { - std::cout << "[cwd=" << cwd << "]; cmd='" << cmd << "\'; args='" << args << "'; envs='" - << envs << "'\n"; - } - - // See CreateProcess() WIN32 documentation for details. - constexpr std::size_t CMDLINE_LENGTH = 32767; - - // Build lpCommandLine parameter. - std::string cmdline = quote_string(cmd); - if(not args.empty()) - cmdline += " " + args; - - // clang-format off - if(cmdline.size() > CMDLINE_LENGTH) - MIGRAPHX_THROW("Command line too long, required maximum " + - std::to_string(CMDLINE_LENGTH) + " characters."); - // clang-format on - - if(cmdline.size() < CMDLINE_LENGTH) - cmdline.resize(CMDLINE_LENGTH, '\0'); - - // Build lpEnvironment parameter. - std::vector environment{}; - if(not envs.empty()) - { - std::istringstream iss{envs}; - std::string str; - while(iss >> str) - { - environment.insert(environment.end(), str.begin(), str.end()); - environment.push_back('\0'); - } - environment.push_back('\0'); - } - - try - { - STARTUPINFO info; - PROCESS_INFORMATION process_info; - - pipe input{}; - pipe output{}; - - ZeroMemory(&info, sizeof(STARTUPINFO)); - info.cb = sizeof(STARTUPINFO); - info.hStdError = output.get_write_handle(); - info.hStdOutput = output.get_write_handle(); - info.hStdInput = input.get_read_handle(); - info.dwFlags |= STARTF_USESTDHANDLES; - - ZeroMemory(&process_info, sizeof(process_info)); - - if(CreateProcess(cmd.c_str(), - cmdline.data(), - nullptr, - nullptr, - TRUE, - 0, - environment.empty() ? nullptr : environment.data(), - cwd.empty() ? nullptr : static_cast(cwd.c_str()), - &info, - &process_info) == FALSE) - { - MIGRAPHX_THROW("Error creating process (" + std::to_string(GetLastError()) + ")"); - } - - CloseHandle(process_info.hThread); - - if(not output.close_write_handle()) - MIGRAPHX_THROW("Error closing STDOUT handle for writing (" + - std::to_string(GetLastError()) + ")"); - - if(not input.close_read_handle()) - MIGRAPHX_THROW("Error closing STDIN handle for reading (" + - std::to_string(GetLastError()) + ")"); - - f(input, output); - - if(not input.close_write_handle()) - MIGRAPHX_THROW("Error closing STDIN handle for writing (" + - std::to_string(GetLastError()) + ")"); - - WaitForSingleObject(process_info.hProcess, INFINITE); - - DWORD status{}; - GetExitCodeProcess(process_info.hProcess, &status); - - CloseHandle(process_info.hProcess); - - return static_cast(status); - } - // cppcheck-suppress catchExceptionByValue - catch(DWORD error) - { - MIGRAPHX_THROW("Error spawning process (" + std::to_string(error) + ")"); - } -} - -// clang-format off -int exec(const std::string& cmd, const std::string& cwd, const std::string& args, - const std::string& envs, HANDLE std_out) -{ - TCHAR buffer[MIGRAPHX_PROCESS_BUFSIZE]; - return (std_out == nullptr or std_out == INVALID_HANDLE_VALUE) - ? GetLastError() : exec(cmd, cwd, args, envs, - [&](const pipe&, const pipe& out) { - for(;;) - { - auto [more_data, bytes_read] = out.read(buffer, MIGRAPHX_PROCESS_BUFSIZE); - if(bytes_read == 0) - break; - if(WriteFile(std_out, buffer, bytes_read, nullptr, nullptr) == FALSE) - break; - if(not more_data) - break; - } - }); -} - -int exec(const std::string& cmd, const std::string& cwd, const std::string& args, - const std::string& envs, std::function std_in) -{ - return exec(cmd, cwd, args, envs, - [&](const pipe& input, const pipe&) { - std_in([&](const char* buffer, std::size_t n) { input.write(buffer, n); }); - }); -} -// clang-format on - -#endif - -struct process_impl -{ - std::string args{}; - std::string envs{}; - std::string command{}; - fs::path cwd{}; - - std::string get_command() const - { - std::string result; - if(not cwd.empty()) - result += "cd " + cwd.string() + "; "; - if(not envs.empty()) - result += envs + " "; - result += command; - if(not args.empty()) - result += " " + args; - return result; - } - - template - void check_exec(Ts&&... xs) const - { - int ec = migraphx::exec(std::forward(xs)...); - if(ec != 0) - MIGRAPHX_THROW("Command " + get_command() + " exited with status " + - std::to_string(ec)); - } -}; - -process::process(const std::string& cmd, const std::vector& args) - : impl(std::make_unique()) -{ - impl->command = cmd; - if(not args.empty()) - impl->args = join_strings(args, " "); -} - -process::process(process&&) noexcept = default; - -process& process::operator=(process rhs) -{ - std::swap(impl, rhs.impl); - return *this; -} - -process::~process() noexcept = default; - -process& process::cwd(const fs::path& p) -{ - impl->cwd = p; - return *this; -} - -process& process::env(const std::vector& envs) -{ - if(not envs.empty()) - { - impl->envs = join_strings(envs, " "); - } - return *this; -} - -void process::read(const writer& output) const -{ -#ifdef _WIN32 - // clang-format off - constexpr std::string_view filename = "stdout"; - auto tmp = tmp_dir{}; - HANDLE handle = CreateFile((tmp.path / filename).string().c_str(), - GENERIC_READ | GENERIC_WRITE, - 0, - nullptr, - CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, - nullptr); - impl->check_exec(impl->command, impl->cwd.string(), impl->args, impl->envs, - handle == nullptr or handle == INVALID_HANDLE_VALUE ? - GetStdHandle(STD_OUTPUT_HANDLE) : handle); - CloseHandle(handle); - handle = CreateFile((tmp.path / filename).string().c_str(), - GENERIC_READ | GENERIC_WRITE, - 0, - nullptr, - OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, - nullptr); - if(handle == nullptr or handle == INVALID_HANDLE_VALUE) - MIGRAPHX_THROW("Unable to open file: " + (tmp.path / filename)); - auto size = GetFileSize(handle, nullptr); - std::string result(size, '\0'); - if(ReadFile(handle, result.data(), size, nullptr, nullptr) == FALSE) - MIGRAPHX_THROW("Failed reading file: " + (tmp.path / filename)); - CloseHandle(handle); - // clang-format on -#else - std::stringstream ss; - impl->check_exec(impl->get_command(), redirect_to(ss)); - auto result = ss.str(); -#endif - output(result.data(), result.size()); -} - -void process::exec() -{ -#ifndef _WIN32 - impl->check_exec(impl->get_command(), redirect_to(std::cout)); -#else - // clang-format off - impl->check_exec(impl->command, impl->cwd.string(), impl->args, impl->envs, - GetStdHandle(STD_OUTPUT_HANDLE)); - // clang-format on -#endif -} - -void process::write(std::function pipe_in) -{ -#ifndef _WIN32 - impl->check_exec(impl->get_command(), std::move(pipe_in)); -#else - // clang-format off - impl->check_exec(impl->command, impl->cwd.string(), - impl->args, impl->envs, std::move(pipe_in)); - // clang-format on -#endif -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/program.cpp b/docker/rocm/migraphx/program.cpp deleted file mode 100644 index 0170809c1..000000000 --- a/docker/rocm/migraphx/program.cpp +++ /dev/null @@ -1,1331 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using milliseconds = std::chrono::duration; - -struct mark_instruction_target -{ - std::size_t target_id = 0; - std::string name() const { return "mark_instruction_target"; } - void apply(module& m) const - { - for(auto& ins : m) - ins.set_target_id(target_id); - } -}; - -struct program_impl -{ - // A map is used to keep references to modules of the program - std::unordered_map modules; - std::vector contexts; - std::vector targets; -}; - -program::program() : impl(std::make_unique()) { this->create_module("main"); } -program::program(module m) : impl(std::make_unique()) -{ - this->create_module("main", std::move(m)); -} - -program::program(program&&) noexcept = default; -program::~program() noexcept = default; - -// copy constructor -program::program(const program& p) { assign(p); } - -// copy assignment operator -program& program::operator=(program p) -{ - std::swap(p.impl, this->impl); - return *this; -} - -void program::assign(const program& p) -{ - if(not impl) - { - impl = std::make_unique(); - } - - *impl = *p.impl; - - // build a map from old ins to new ins - // Build a map from old module to new module - std::unordered_map mod_map; - std::transform( - impl->modules.begin(), - impl->modules.end(), - std::inserter(mod_map, mod_map.begin()), - [&](auto&& xp) { return std::make_pair(&p.impl->modules.at(xp.first), &xp.second); }); - - std::unordered_map ins_map; - for(auto&& pp : mod_map) - { - auto old_ins = iterator_for(*pp.first); - auto new_ins = iterator_for(*pp.second); - std::transform(old_ins.begin(), - old_ins.end(), - new_ins.begin(), - std::inserter(ins_map, ins_map.begin()), - [](auto x, auto y) { return std::make_pair(x, y); }); - } - - // Update all references from all modules - for(auto&& mp : impl->modules) - { - for(auto ins : iterator_for(mp.second)) - instruction::replace_refs(ins, ins_map, mod_map); - } -} - -shape program::get_parameter_shape(std::string name) const -{ - const auto* mm = this->get_main_module(); - return mm->get_parameter_shape(std::move(name)); -} - -std::vector program::get_parameter_names() const -{ - const auto* mm = this->get_main_module(); - return mm->get_parameter_names(); -} - -instruction_ref program::get_parameter(std::string name) const -{ - const auto* mm = this->get_main_module(); - return mm->get_parameter(std::move(name)); -} - -std::unordered_map program::get_parameter_shapes() const -{ - const auto* mm = this->get_main_module(); - return mm->get_parameter_shapes(); -} - -std::size_t program::size() const { return impl->modules.size(); } - -std::vector program::get_output_shapes() const -{ - const auto* mm = this->get_main_module(); - return mm->get_output_shapes(); -} - -context& program::get_context() const -{ - assert(impl->contexts.size() == 1); - return impl->contexts.front(); -} - -instruction_ref program::validate() const -{ - const auto* mm = this->get_main_module(); - return mm->validate(); -} - -target_assignments program::get_target_assignments(const std::vector& targets, - assignment_options options) -{ - const auto m = options.metric; - - target_assignments p; - - const auto* mod = get_main_module(); - std::vector> target_subgraphs; - target_subgraphs.reserve(targets.size()); - std::transform(targets.begin(), - targets.end(), - std::back_inserter(target_subgraphs), - [&](const auto& t) { return std::make_pair(t, t.find_supported(mod, m)); }); - - for(const auto ins : iterator_for(*mod)) - { - if(contains(p, ins)) - { - continue; - } - - for(const auto& [target, subgraph] : target_subgraphs) - { - // can't pass a structured binding into lambda in C++17 so create a variable for it - const auto& t = target; - for(const auto& segment : subgraph) - { - const auto& instructions = segment.instructions; - if(not contains(instructions, ins)) - { - continue; - } - std::transform(instructions.begin(), - instructions.end(), - std::inserter(p, p.end()), - [&](auto instr) { return std::make_pair(instr, t.name()); }); - } - } - } - return p; -} - -bool program::is_compiled() const { return not this->impl->contexts.empty(); } - -void program::compile(const std::vector& targets, std::vector compile_opts) -{ - // Gather all the target roots - std::unordered_multimap roots; - auto mods = this->get_modules(); - for(const auto* mod : mods) - { - for(const auto& ins : *mod) - { - if(ins.name() != "run_on_target") - continue; - auto v = ins.get_operator().to_value(); - module_ref root = ins.module_inputs().front(); - std::size_t root_target_id = v.at("target_id").to(); - assert(root_target_id < targets.size()); - roots.insert({root_target_id, root}); - } - } - - auto trace = tracer{}; - // TODO: Add tracer based on compile options - if(enabled(MIGRAPHX_TRACE_COMPILE{})) - trace = tracer{std::cout}; - - trace(*this); - trace(); - // It is assumed that all instructions outside of any root module would run on "ref" target - // Ref target may or may not be passed as one of the target for the "compile()". - // If it is not passed, Create one and add context of it into the map. - auto target_idx = [&](const std::string& t_name) { - return static_cast( - std::find_if( - targets.begin(), targets.end(), [&](const auto& t) { return t.name() == t_name; }) - - targets.begin()); - }; - - std::size_t ref_target_id = target_idx("ref"); - if(ref_target_id == targets.size()) - { - this->impl->contexts.resize(targets.size() + 1); - this->impl->contexts[ref_target_id] = migraphx::make_target("ref").get_context(); - // users could pass lessers compile_ops than targets, in that case use default compile_opts - compile_opts.resize(targets.size() + 1, migraphx::compile_options{}); - } - else - { - this->impl->contexts.resize(targets.size()); - compile_opts.resize(targets.size(), migraphx::compile_options{}); - } - // mark all the instruction as ref target first, later change target_id based on root-target - run_passes(*this, {mark_instruction_target{ref_target_id}}); - - // Run passes on each root target - for(const auto i : range(targets.size())) - { - const auto& root_target = targets.at(i); - auto root_target_id = i; - auto root_modules_range = roots.equal_range(root_target_id); - this->impl->contexts[root_target_id] = root_target.get_context(); - for(const auto& [id, current_mod] : range(root_modules_range)) - { - auto passes = root_target.get_passes(this->impl->contexts[root_target_id], - compile_opts[root_target_id]); - passes.push_back(mark_instruction_target{static_cast(root_target_id)}); - run_passes(*this, current_mod, passes, trace); - - auto invalid = current_mod->validate(); - if(invalid != current_mod->end()) - { - MIGRAPHX_THROW("Invalid module " + current_mod->name() + - " from compilation at instruction " + - std::to_string(std::distance(current_mod->begin(), invalid))); - } - auto dangling = current_mod->find_dangling_reference(); - if(dangling != current_mod->end()) - { - auto index = std::distance(current_mod->begin(), dangling); - MIGRAPHX_THROW("Dangling reference in module " + current_mod->name() + - " from instruction " + std::to_string(index)); - } - } - } - this->finalize(); -} - -void program::compile(const target& t, compile_options options) -{ - // todo: combine with multi-target compile method - assert(not this->is_compiled()); - this->impl->targets = {t}; - this->impl->contexts = {t.get_context()}; - - if(enabled(MIGRAPHX_TRACE_COMPILE{})) - options.trace = tracer{std::cout}; - - options.trace(*this); - options.trace(); - auto&& passes = t.get_passes(this->impl->contexts.front(), options); - run_passes(*this, passes, options.trace); - auto mods = this->get_modules(); - // Validate and finalize - for(const auto& mod : reverse(mods)) - { - auto invalid = mod->validate(); - if(invalid != mod->end()) - { - MIGRAPHX_THROW("Invalid module " + mod->name() + " from compilation at instruction " + - std::to_string(std::distance(mod->begin(), invalid))); - } - auto dangling = mod->find_dangling_reference(); - if(dangling != mod->end()) - { - auto index = std::distance(mod->begin(), dangling); - MIGRAPHX_THROW("Dangling reference in module " + mod->name() + " from instruction " + - std::to_string(index)); - } - mod->finalize(this->impl->contexts); - } -} - -void program::finalize() -{ - auto* mm = this->get_main_module(); - mm->finalize(this->impl->contexts); -} - -template -std::string classify(T x) -{ - switch(std::fpclassify(static_cast(x))) - { - case FP_INFINITE: return "inf"; - case FP_NAN: return "nan"; - case FP_NORMAL: return "normal"; - case FP_SUBNORMAL: return "subnormal"; - case FP_ZERO: return "zero"; - default: return "unknown"; - } -} - -void print_statistics(std::ostream& os, const argument& a) -{ - a.visit( - [&](auto t) { - os << "Min value: " << *std::min_element(t.begin(), t.end()) << ", "; - os << "Max value: " << *std::max_element(t.begin(), t.end()) << ", "; - double num_elements = t.size(); - auto mean = std::accumulate(t.begin(), t.end(), 0.0) / num_elements; - auto stddev = std::sqrt( - std::accumulate(t.begin(), - t.end(), - 0.0, - [&](auto r, auto v) { return r + std::pow((v - mean), 2.0); }) / - num_elements); - os << "Mean: " << mean << ", "; - os << "StdDev: " << stddev << "\n"; - }, - [&](const auto& xs) { - for(const auto& x : xs) - { - print_statistics(os, x); - } - }); -} - -std::unordered_set classify_argument(const argument& a) -{ - std::unordered_set result; - a.visit( - [&](auto t) { - for(const auto& x : t) - result.insert(classify(x)); - }, - [&](const auto& xs) { - for(const auto& x : xs) - { - auto r = classify_argument(x); - result.insert(r.begin(), r.end()); - } - }); - return result; -} - -void preview_argument(std::ostream& os, const argument& a) -{ - a.visit( - [&](auto t) { - if(t.size() <= 10) - { - os << t; - } - else - { - os << to_string_range(t.begin(), t.begin() + 5); - os << ", ..., "; - os << to_string_range(t.end() - 5, t.end()); - } - }, - [&](const auto& xs) { - for(const auto& x : xs) - { - os << '{'; - preview_argument(os, x); - os << '}'; - } - }); -} - -// This function currently used only in an Assertion. -// "Almost identical" shapes. To support an MLIR feature, there is a limited -// case where shapes may both be standard but have non-identical strides. -#ifndef NDEBUG -static bool is_compatible_shape(const shape& actual, const shape& expected) -{ - // Check subshapes - if(expected.type() == shape::tuple_type) - return equal(actual.sub_shapes().begin(), - actual.sub_shapes().end(), - expected.sub_shapes().begin(), - &is_compatible_shape); - // Only the expected can be dynamic - if(expected.dynamic()) - return true; - if(actual == expected) - return true; - if(actual.type() != expected.type()) - return false; - // If both shapes are standard and lens match, they are considered compatible - // even if strides are different. - if(actual.standard() and expected.standard()) - return actual.lens() == expected.lens(); - return false; -} -#endif - -template -std::vector generic_eval(const module* mod, - std::vector& ctx, - std::unordered_map params, - std::unordered_map results, - F trace) -{ - assert(mod->validate() == mod->end()); - results.reserve(mod->size() * 2); - std::vector values; - values.reserve(16); - for(auto ins : iterator_for(*mod)) - { - assert(results.find(ins) == results.end()); - const auto& name = ins->name(); - if(name == "@literal") - { - results.emplace(ins, trace(ins, [&] { return ins->get_literal().get_argument(); })); - } - else if(name == "@param") - { - results.emplace( - ins, trace(ins, [&] { - auto param_name = any_cast(ins->get_operator()).parameter; - if(not contains(params, param_name)) - MIGRAPHX_THROW("Parameter not found: " + param_name); - auto param = params[param_name]; - // TODO: may want to check correct number of dimensions and/or was within bounds - if(not ins->get_shape().any_of_dynamic() and - param.get_shape() != ins->get_shape()) - { - MIGRAPHX_THROW("Incorrect shape {" + to_string(param.get_shape()) + - "} for parameter: " + param_name + - " should be: " + to_string(ins->get_shape())); - } - return param; - })); - } - else if(name == "@outline") - { - results.emplace(ins, trace(ins, [&] { return argument{ins->get_shape(), nullptr}; })); - } - else if(name == "@return") - { - std::vector prog_outputs; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(prog_outputs), - [&](instruction_ref i) { - assert(results.find(i) != results.end()); - return results[i]; - }); - - return prog_outputs; - } - else - { - values.resize(ins->inputs().size()); - std::transform( - ins->inputs().begin(), ins->inputs().end(), values.begin(), [&](instruction_ref i) { - assert(results.find(i) != results.end()); - return results[i]; - }); - const auto& mod_args = ins->module_inputs(); - auto module_eval = [&](module_ref smod, - const std::unordered_map& inputs) { - return generic_eval(smod, ctx, inputs, results, trace); - }; - - results.emplace( - ins, trace(ins, [&] { - auto op = ins->normalized_operator(); - if(op.is_context_free()) - return op.compute(ins->get_shape(), values, mod_args, module_eval); - if(ins->get_target_id() >= ctx.size()) - MIGRAPHX_THROW("No context available for " + op.name()); - return op.compute( - ctx[ins->get_target_id()], ins->get_shape(), values, mod_args, module_eval); - })); - } - assert(results.find(ins) != results.end()); - assert(is_compatible_shape(results.at(ins).get_shape(), ins->get_shape())); - } - return {results.at(std::prev(mod->end()))}; -} - -template -std::vector generic_eval(const program& p, - std::vector& ctx, - std::unordered_map params, - F trace) -{ - const module* mm = p.get_main_module(); - return generic_eval(mm, ctx, params, {}, trace); -} - -std::vector program::eval_with_context(std::vector& ctx, - parameter_map params) const -{ - const module* mm = this->get_main_module(); - return generic_eval(mm, ctx, std::move(params), {}, [](auto&&, auto f) { return f(); }); -} - -std::vector program::eval(parameter_map params, execution_environment exec_env) const -{ - auto& contexts = this->impl->contexts; - - auto trace_level = value_of(MIGRAPHX_TRACE_EVAL{}); - std::vector ret; - - if(exec_env.async) - { - assert(contexts.size() == 1); - contexts.front().wait_for(exec_env.queue); - } - - if(trace_level > 0) - { - std::unordered_map ins_out; - // get instruction names - this->print([&](auto x, auto ins_names) { - std::stringstream ss; - instruction::print(ss, x, ins_names); - ins_out[x] = ss.str(); - }); - ret = generic_eval(*this, contexts, std::move(params), [&](instruction_ref ins, auto f) { - const auto& ctx = contexts[ins->get_target_id()]; - ctx.finish(); - std::cout << "Run instruction: " << ins_out.at(ins) << std::endl; - timer t{}; - auto result = f(); - double t1 = t.record(); - ctx.finish(); - double t2 = t.record(); - std::cout << "Time: " << t1 << "ms, " << t2 << "ms" << std::endl; - if(trace_level > 1 and ins->name().front() != '@' and ins->name() != "load" and - not result.empty()) - { - migraphx::argument buffer; - try - { - const target& tgt = this->impl->targets.at(ins->get_target_id()); - buffer = tgt.copy_from(result); - } - catch(const migraphx::exception&) - { - // instruction was run on host then no need to copy buffer from target - buffer = result; - } - catch(...) - { - MIGRAPHX_THROW("MIGraphX program execution with MIGRAPHX_TRACE_EVAL failed.\n"); - } - if(trace_level == 2) - { - std::cout << "Output has " << to_string_range(classify_argument(buffer)) - << std::endl; - std::cout << "Output: "; - preview_argument(std::cout, buffer); - std::cout << std::endl; - print_statistics(std::cout, buffer); - } - else - { - std::cout << "Output: " << buffer << std::endl; - } - } - return result; - }); - } - else - { - ret = generic_eval(*this, contexts, std::move(params), [&](auto&&, auto f) { return f(); }); - } - - if(exec_env.async) - { - assert(contexts.size() == 1); - contexts.front().finish_on(exec_env.queue); - } - - return ret; -} - -void program::finish() const -{ - for(const auto& ctx : this->impl->contexts) - ctx.finish(); -} - -std::string get_migraphx_version() -{ - std::stringstream ss; - ss << std::to_string(MIGRAPHX_VERSION_MAJOR) << "." << std::to_string(MIGRAPHX_VERSION_MINOR) - << "." << std::to_string(MIGRAPHX_VERSION_PATCH); - return ss.str(); -} - -/* -program file version is for the data structure or format of the MXR file. Version should be bumped -if any changes occur to the format of the MXR file. -*/ -const int program_file_version = 7; - -value program::to_value() const -{ - value result; - result["version"] = program_file_version; - result["migraphx_version"] = get_migraphx_version(); - result["targets"] = migraphx::to_value(this->impl->targets); - result["contexts"] = migraphx::to_value(this->impl->contexts); - value module_vals = value::object{}; - std::unordered_map names; - for(auto& mod : this->get_modules()) - { - value mod_val; - value nodes; - mod_val["name"] = mod->name(); - names = mod->print( - [&](auto ins, auto ins_names) { - value node; - node["output"] = ins_names.at(ins); - node["name"] = ins->name(); - node["shape"] = migraphx::to_value(ins->get_shape()); - node["normalized"] = ins->is_normalized(); - if(ins->name() == "@literal") - node["literal"] = migraphx::to_value(ins->get_literal()); - node["operator"] = ins->get_operator().to_value(); - std::vector inputs; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(inputs), - [&](auto i) { - assert(contains(ins_names, i)); - return ins_names.at(i); - }); - node["inputs"] = inputs; - auto module_args = ins->module_inputs(); - if(not module_args.empty()) - { - std::vector module_inputs; - std::transform(module_args.begin(), - module_args.end(), - std::back_inserter(module_inputs), - [&](auto mod_ref) { return mod_ref->name(); }); - node["module_inputs"] = module_inputs; - } - - nodes.push_back(node); - }, - names); - mod_val["nodes"] = nodes; - - module_vals[mod->name()] = mod_val; - } - - result["modules"] = module_vals; - - return result; -} - -static void mod_from_val(module_ref mod, - const value& v, - std::unordered_map& instructions, - const std::unordered_map& map_mods) -{ - const auto& module_val = v.at(mod->name()); - for(const value& node : module_val.at("nodes")) - { - instruction_ref output; - auto name = node.at("name").to(); - auto fields = node.at("operator"); - auto normalized = node.at("normalized").to(); - - if(name == "@param") - { - output = mod->insert_parameter(mod->end(), - fields["parameter"].to(), - migraphx::from_value(node.at("shape"))); - } - else if(name == "@literal") - { - output = - mod->insert_literal(mod->end(), migraphx::from_value(node.at("literal"))); - } - else - { - auto op = make_op(name, fields); - std::vector inputs; - std::transform(node.at("inputs").begin(), - node.at("inputs").end(), - std::back_inserter(inputs), - [&](const value& i) { - auto i_name = i.to(); - assert(contains(instructions, i_name)); - return instructions.at(i_name); - }); - - std::vector module_inputs; - if(node.contains("module_inputs")) - { - std::transform(node.at("module_inputs").begin(), - node.at("module_inputs").end(), - std::back_inserter(module_inputs), - [&](const value& i) { return map_mods.at(i.to()); }); - - for(const auto& smod : module_inputs) - { - mod_from_val(smod, v, instructions, map_mods); - } - } - - if(name == "@return") - { - output = mod->add_return(inputs); - } - else if(module_inputs.empty()) - { - output = mod->insert_instruction(mod->end(), op, inputs); - } - else - { - output = mod->insert_instruction(mod->end(), op, inputs, module_inputs); - } - } - output->set_normalized(normalized); - instructions[node.at("output").to()] = output; - } -} - -void program::from_value(const value& v) -{ - auto version = v.at("version").to(); - if(version != program_file_version) - { - MIGRAPHX_THROW( - "Error: Program version mismatch. MXR file was created using program file version: " + - std::to_string(version) + ", while installed MIGraphX is using program file version: " + - std::to_string(program_file_version) + - ", Try regenerating MXR file using installed MIGraphX and running again."); - } - - auto migx_version = v.at("migraphx_version").to(); - if(migx_version != get_migraphx_version()) - { - std::cout << "[WARNING]: MXR File was created using MIGraphX version: " << migx_version - << ", while installed MIGraphX is at version: " << get_migraphx_version() - << ", operators implementation could be mismatched.\n"; - } - - migraphx::from_value(v.at("targets"), this->impl->targets); - - for(auto i : range(this->impl->targets.size())) - { - this->impl->contexts.push_back(this->impl->targets[i].get_context()); - this->impl->contexts.back().from_value(v.at("contexts")[i]); - } - - auto module_vals = v.at("modules"); - for(const auto& vv : module_vals) - { - const auto& name = vv.get_key(); - if(name == "main") - continue; - impl->modules.emplace(name, name); - } - std::unordered_map map_mods; - std::transform(impl->modules.begin(), - impl->modules.end(), - std::inserter(map_mods, map_mods.end()), - [&](auto&& pp) { return std::make_pair(pp.first, &pp.second); }); - - std::unordered_map map_insts; - auto* mm = get_main_module(); - mod_from_val(mm, module_vals, map_insts, map_mods); - - // Finalize a compiled model - if(not this->impl->contexts.empty()) - this->finalize(); -} - -double common_average(const std::vector& v) -{ - std::size_t n = v.size() / 4; - double total = std::accumulate(v.begin() + n, v.end() - n, 0.0); - return total / std::distance(v.begin() + n, v.end() - n); -} - -double mean(const std::vector& v) -{ - double total = std::accumulate(v.begin(), v.end(), 0.0); - return total / v.size(); -} - -double median(const std::vector& v) -{ - size_t mid = v.size() / 2; - if(v.size() % 2 == 0) - { - return (v[mid - 1] + v[mid]) / 2.0; - } - else - { - return v[mid]; - } -} - -double percentile(const std::vector& v, double percentile) -{ - size_t index = (percentile * (v.size() - 1)); - return v[index]; -} - -std::string perf_group(instruction_ref ins, bool detailed) -{ - std::string result; - auto attr = ins->get_operator().attributes(); - if(attr.contains("group")) - result = attr.at("group").to(); - else - result = ins->get_operator().name(); - if(detailed) - { - result += "<" + ins->get_shape().type_string(); - result += "(" + shape::to_sizes_string(to_shapes(ins->inputs())) + ")>"; - } - return result; -} - -void program::mark(const parameter_map& params, marker m) -{ - auto& ctx = this->impl->contexts; - // Run once by itself - eval(params); - this->finish(); - m.mark_start(*this); - generic_eval(*this, ctx, params, [&](auto ins, auto f) { - argument result; - m.mark_start(ins); - result = f(); - m.mark_stop(ins); - return result; - }); - m.mark_stop(*this); -} - -void program::perf_report( - std::ostream& os, std::size_t n, parameter_map params, std::size_t batch, bool detailed) const -{ - auto& ctx = this->impl->contexts; - // Run once by itself - eval(params); - this->finish(); - // Run and time entire program - std::vector total_vec; - total_vec.reserve(n); - for(std::size_t i = 0; i < n; i++) - { - total_vec.push_back(time([&] { - eval(params); - this->finish(); - })); - } - std::sort(total_vec.begin(), total_vec.end()); - std::unordered_map> ins_vec; - // Fill the map - generic_eval(*this, ctx, params, [&](auto ins, auto) { - ins_vec[ins].reserve(n); - return argument{ins->get_shape(), nullptr}; - }); - - // Run and time each instruction - for(std::size_t i = 0; i < n; i++) - { - generic_eval(*this, ctx, params, [&](auto ins, auto f) { - argument result; - ins_vec[ins].push_back(time([&] { - result = f(); - this->impl->contexts[ins->get_target_id()].finish(); - })); - return result; - }); - } - for(auto&& p : ins_vec) - std::sort(p.second.begin(), p.second.end()); - // Run and time implicit overhead - std::vector overhead_vec; - overhead_vec.reserve(n); - for(std::size_t i = 0; i < n; i++) - { - overhead_vec.push_back(time([&] { dry_run(params); })); - } - double total_time = common_average(total_vec); - double min_time = total_vec.front(); - double max_time = total_vec.back(); - double mean_time = mean(total_vec); - double median_time = median(total_vec); - double percentile_90_time = percentile(total_vec, 0.90); - double percentile_95_time = percentile(total_vec, 0.95); - double percentile_99_time = percentile(total_vec, 0.99); - double rate = 1000.0 / total_time; - double overhead_time = common_average(overhead_vec); - double overhead_percent = overhead_time * 100.0 / total_time; - double total_instruction_time = 0.0; - std::unordered_map op_times; - std::unordered_map op_n; - for(auto&& p : ins_vec) - { - double avg = common_average(p.second); - op_times[perf_group(p.first, detailed)] += avg; - total_instruction_time += avg; - op_n[perf_group(p.first, detailed)]++; - } - double calculate_overhead_time = total_time - total_instruction_time; - double calculate_overhead_percent = calculate_overhead_time * 100.0 / total_time; - - std::unordered_map names; - this->print(names, [&](auto ins, auto ins_names) { - instruction::print(std::cout, ins, ins_names); - - // skip return instruction - if(ins->name() == "@return") - return; - - double avg = common_average(ins_vec[ins]); - double percent = std::ceil(100.0 * avg / total_instruction_time); - os << ": " << avg << "ms, " << percent << "%"; - os << std::endl; - }); - - os << std::endl; - os << "Summary:" << std::endl; - std::vector> op_times_sorted; - std::transform( - op_times.begin(), op_times.end(), std::back_inserter(op_times_sorted), [&](auto p) { - auto&& name = p.first; - return std::make_tuple(p.second, op_n.at(name), name); - }); - std::sort(op_times_sorted.begin(), op_times_sorted.end(), std::greater<>{}); - for(auto&& [avg, nn, name] : op_times_sorted) - { - double percent = std::ceil(100.0 * avg / total_instruction_time); - double per_ins = avg / nn; - os << name << ": " << avg << "ms / " << nn << " = " << per_ins << "ms, " << percent << "%" - << std::endl; - } - - os << std::endl; - - os << "Batch size: " << batch << std::endl; - os << "Rate: " << rate * batch << " inferences/sec" << std::endl; - os << "Total time: " << total_time << "ms "; - os << "(Min: " << min_time << "ms, "; - os << "Max: " << max_time << "ms, "; - os << "Mean: " << mean_time << "ms, "; - os << "Median: " << median_time << "ms)" << std::endl; - os << "Percentiles (90%, 95%, 99%): ("; - os << percentile_90_time << "ms, " << percentile_95_time << "ms, " << percentile_99_time - << "ms)" << std::endl; - os << "Total instructions time: " << total_instruction_time << "ms" << std::endl; - os << "Overhead time: " << overhead_time << "ms" - << ", " << calculate_overhead_time << "ms" << std::endl; - os << "Overhead: " << std::round(overhead_percent) << "%" - << ", " << std::round(calculate_overhead_percent) << "%" << std::endl; -} - -void program::debug_print() const { std::cout << *this << std::endl; } -void program::debug_print(instruction_ref ins) const -{ - std::unordered_map names; - if(std::any_of(this->impl->modules.begin(), this->impl->modules.end(), [&](const auto& pp) { - return is_end(pp.second.end(), ins); - })) - { - std::cout << "End instruction" << std::endl; - return; - } - else if(std::none_of(this->impl->modules.begin(), - this->impl->modules.end(), - [&](const auto& pp) { return pp.second.has_instruction(ins); })) - { - std::cout << "Instruction not part of program" << std::endl; - return; - } - - this->print(names, [&](auto x, auto ins_names) { - if(x == ins) - { - instruction::print(std::cout, x, ins_names); - std::cout << std::endl; - } - }); -} - -void program::print( - std::unordered_map& names, - const std::function)>& - print_func) const -{ - for(const auto& pp : this->impl->modules) - { - names = pp.second.print(print_func, names); - } -} - -void program::print( - const std::function)>& print_func) const -{ - std::unordered_map names; - this->print(names, print_func); -} - -void program::print_graph(std::ostream& os, bool brief) const -{ - const auto* mm = this->get_main_module(); - mm->print_graph(os, brief); -} - -void program::print_py(std::ostream& os) const -{ - auto vec_modules = this->get_modules(); - std::unordered_map names; - os << "p = migraphx.program()\n"; - for(auto& mod : vec_modules) - { - std::string var_name = "m"; - if(mod->name() != "main") - var_name += mod->name(); - os << var_name << " = "; - if(mod->name() == "main") - os << "p.get_main_module()"; - else - os << "p.create_module(\"" << mod->name() << "\");"; - os << std::endl; - names = mod->print_py(os, var_name, names); - os << std::endl; - } -} - -void program::print_cpp(std::ostream& os) const -{ - auto vec_modules = this->get_modules(); - std::unordered_map names; - os << "migraphx::program p;\n"; - for(auto& mod : vec_modules) - { - std::string var_name = "m" + mod->name(); - os << "migraphx::module_ref " << var_name << " = "; - if(mod->name() == "main") - os << "p.get_main_module();"; - else - os << "p.create_module(\"" << mod->name() << "\");"; - os << std::endl; - names = mod->print_cpp(os, var_name, names); - os << std::endl; - } -} - -void program::dry_run(std::unordered_map params) const -{ - auto& ctx = this->impl->contexts; - generic_eval(*this, ctx, std::move(params), [](auto ins, auto&&...) { - return argument{ins->get_shape(), nullptr}; - }); -} - -void program::annotate(std::ostream& os, const std::function& a) const -{ - for(auto& pp : this->impl->modules) - { - std::cout << pp.first << ":" << std::endl; - pp.second.annotate(os, a); - } -} - -const module* program::get_module(const std::string& name) const { return &impl->modules.at(name); } - -module* program::create_module(const std::string& name) -{ - - assert(not contains(impl->modules, name)); - auto r = impl->modules.emplace(name, name); - return &(r.first->second); -} - -module* program::create_module(const std::string& name, module m) -{ - assert(not contains(impl->modules, name)); - m.set_name(name); - auto r = impl->modules.emplace(name, std::move(m)); - return &(r.first->second); -} - -module* program::get_module(const std::string& name) { return &impl->modules.at(name); } - -module* program::get_main_module() { return get_module("main"); } - -const module* program::get_main_module() const { return get_module("main"); } - -template -std::vector generic_get_modules(T* mm) -{ - std::vector vec_modules; - vec_modules.push_back(mm); - auto sub_modules = mm->get_sub_modules(); - vec_modules.insert(vec_modules.end(), sub_modules.begin(), sub_modules.end()); - return vec_modules; -} - -template -void generic_get_unused_modules(Map& m, const std::vector& mods, OutputIterator out) -{ - std::unordered_set used; - std::transform(mods.begin(), mods.end(), std::inserter(used, used.end()), [](auto&& mod) { - return mod->name(); - }); - transform_if( - m.begin(), - m.end(), - out, - [&](auto&& pp) { return not contains(used, pp.first); }, - [](auto&& pp) { return &pp.second; }); -} - -std::vector program::get_modules() const -{ - auto result = generic_get_modules(this->get_main_module()); - generic_get_unused_modules(impl->modules, result, std::back_inserter(result)); - return result; -} - -std::vector program::get_modules() -{ - auto result = generic_get_modules(this->get_main_module()); - generic_get_unused_modules(impl->modules, result, std::back_inserter(result)); - return result; -} - -template -void generic_insert_module_tree(Module* pm, Map& m) -{ - for(auto* sm : pm->get_sub_modules(true)) - { - m.insert(std::make_pair(sm, pm)); - generic_insert_module_tree(sm, m); - } -} - -std::unordered_multimap program::get_module_tree() -{ - std::unordered_multimap result; - generic_insert_module_tree(this->get_main_module(), result); - return result; -} - -template -bool is_unused_module(Map& m, const std::vector& mods, const std::string& name) -{ - bool is_unused = false; - generic_get_unused_modules(m, mods, make_function_output_iterator([&](auto* mod) { - if(mod->name() == name) - is_unused = true; - })); - return is_unused; -} - -template -bool references_instruction(Map& m, const instruction& ins, const std::string& name) -{ - return std::any_of(m.begin(), m.end(), [&](auto&& p) { - if(p.first == name) - return false; - return std::any_of(p.second.begin(), p.second.end(), [&](auto&& i) { - return std::any_of(i.inputs().begin(), i.inputs().end(), [&](auto&& j) { - return std::addressof(*j) == std::addressof(ins); - }); - }); - }); -} - -void program::remove_module(const std::string& name) -{ - // cppcheck-suppress assertWithSideEffect - assert(is_unused_module(impl->modules, generic_get_modules(this->get_main_module()), name) and - "Module used in program"); - assert(std::none_of( - impl->modules.at(name).begin(), - impl->modules.at(name).end(), - [&](auto&& ins) { return references_instruction(impl->modules, ins, name); }) and - "Instruction referenced in another module"); - - // if an instruction has an input out side of the current module, need to remove - // the instruction from its input's outputs - auto& mod = impl->modules.at(name); - for(auto ins : iterator_for(mod)) - { - auto inputs = ins->inputs(); - for(auto in : inputs) - { - if(not mod.has_instruction(in)) - { - in->remove_output(ins); - } - } - } - - impl->modules.erase(name); -} - -void program::rename_module(const std::string& old_name, const std::string& new_name) -{ - assert(old_name != new_name); - assert(contains(impl->modules, old_name)); - assert(not contains(impl->modules, new_name)); - auto node = impl->modules.extract(old_name); - node.key() = new_name; - node.mapped().set_name(new_name); - impl->modules.insert(std::move(node)); -} - -void program::remove_unused_modules() -{ - std::vector unused; - generic_get_unused_modules( - impl->modules, generic_get_modules(this->get_main_module()), std::back_inserter(unused)); - for(const auto* m : unused) - this->remove_module(m->name()); -} - -program& program::sort() -{ - std::queue mqueue; - mqueue.push(get_main_module()); - while(not mqueue.empty()) - { - module_ref current_mod = mqueue.front(); - current_mod->sort(); - mqueue.pop(); - auto child_mods = current_mod->get_sub_modules(true); - for(auto& sub_mod : child_mods) - { - mqueue.push(sub_mod); - } - } - return *this; -} - -bool operator==(const program& x, const program& y) { return to_string(x) == to_string(y); } - -std::ostream& operator<<(std::ostream& os, const program& p) -{ - auto vec_modules = p.get_modules(); - std::unordered_map names; - for(auto& mod : vec_modules) - { - os << "module: \"" << mod->name() << "\"" << std::endl; - names = mod->print( - [&](auto ins, auto ins_names) { - instruction::print(os, ins, ins_names); - os << std::endl; - }, - names); - os << std::endl; - } - - return os; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/promote_literals.cpp b/docker/rocm/migraphx/promote_literals.cpp deleted file mode 100644 index 2dfea9ca7..000000000 --- a/docker/rocm/migraphx/promote_literals.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void promote_literals::apply(module_pass_manager& mpm) const -{ - module& m = mpm.get_module(); - module_ref root_module = mpm.get_root_module(); - if(m == *root_module) - return; - - for(auto ins : iterator_for(m)) - { - if(ins->name() == "@literal") - { - auto new_lit = root_module->add_literal(ins->get_literal()); - auto ins_outputs = ins->outputs(); - for(auto out_ins : ins_outputs) - { - out_ins->replace_argument(out_ins, ins, new_lit); - } - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/propagate_constant.cpp b/docker/rocm/migraphx/propagate_constant.cpp deleted file mode 100644 index adad621ce..000000000 --- a/docker/rocm/migraphx/propagate_constant.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_PROPAGATE_CONSTANT) - -bool skip_propagate(instruction_ref ins) -{ - if(contains({"contiguous", "dequantizelinear", "reshape"}, ins->name())) - return skip_propagate(ins->inputs().front()); - if(ins->name() == "unpack_int4") - return true; - auto&& s = ins->get_shape(); - if(s.broadcasted() and s.element_space() < s.elements()) - return true; - auto alias = instruction::get_output_alias(ins, true); - if(alias != ins) - return skip_propagate(alias); - if(ins->is_undefined()) - return true; - return false; -} - -bool is_const_ins(instruction_ref ins, const std::unordered_set& skip_ops) -{ - return ins->can_eval() and not skip_propagate(ins) and - skip_ops.find(ins->name()) == skip_ops.end(); -} - -argument as_packed(const argument& c) -{ - if(c.get_shape().packed()) - return c; - auto s = c.get_shape().with_lens(c.get_shape().lens()); - argument result; - c.visit([&](auto x) { result = literal{s, x.begin(), x.end()}.get_argument(); }); - return result; -} - -void propagate_constant::apply(module& m) const -{ - std::unordered_set const_instrs; - auto last = std::prev(m.end()); - - // Find instructions that can be evaluated to a literal - for(auto i : iterator_for(m)) - { - const bool is_const = is_const_ins(i, skip_ops); - if(is_const and i != last) - continue; - - if(i == last and is_const) - { - const_instrs.insert(i); - } - else - { - std::copy_if(i->inputs().begin(), - i->inputs().end(), - std::inserter(const_instrs, const_instrs.begin()), - [&](const instruction_ref ins) { - return is_const_ins(ins, skip_ops) and ins->name() != "@literal"; - }); - } - } - - // Compute literals in parallel - std::vector const_instrs_vec{const_instrs.begin(), const_instrs.end()}; - std::vector literals(const_instrs_vec.size()); - std::size_t grainsize = 1; -#if !MIGRAPHX_HAS_EXECUTORS - std::size_t n = std::max(2048 / std::thread::hardware_concurrency(), 1); - grainsize = const_instrs_vec.size() / n; -#endif - simple_par_for(const_instrs_vec.size(), grainsize, [&](const auto i) { - literals[i] = as_packed(const_instrs_vec[i]->eval()); - }); - - // Replace instructions in m - for(size_t i = 0; i < const_instrs_vec.size(); i++) - { - if(not literals[i].empty()) - { - if(enabled(MIGRAPHX_TRACE_PROPAGATE_CONSTANT{})) - { - std::cout << "Constant replace: " << std::endl; - std::vector inss; - fix([&](auto self, auto ins) { - if(contains(inss, ins)) - return; - for(auto input : ins->inputs()) - self(input); - inss.push_back(ins); - })(const_instrs_vec[i]); - m.debug_print(inss); - } - assert(literals[i].get_shape().lens() == const_instrs_vec[i]->get_shape().lens()); - assert(literals[i].get_shape().bytes() <= const_instrs_vec[i]->get_shape().bytes()); - auto l = m.add_literal(literals[i].get_shape(), literals[i].data()); - m.replace_instruction(const_instrs_vec[i], l); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/quantization.cpp b/docker/rocm/migraphx/quantization.cpp deleted file mode 100644 index 4c96d233f..000000000 --- a/docker/rocm/migraphx/quantization.cpp +++ /dev/null @@ -1,211 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_8BITS_QUANTIZATION_PARAMS) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_QUANTIZATION) - -tracer quant_tracer() -{ - if(enabled(MIGRAPHX_TRACE_QUANTIZATION{})) - return tracer{std::cout}; - - return tracer{}; -}; - -// This function is to convert any instructions specified in the input -// from double or float to float16 by inserting a convert operator. -// For the conversion, there could be cases of overflowing or underflowing, but it -// is uncommon. Run optimize_module() before converting to fp16 to const eval and fold in FP32 to -// avoid loss of precision. -void quantize_fp16(program& prog, const std::vector& ins_names) -{ - run_passes(prog, - {normalize_ops{}, - optimize_module{{"quantizelinear", "dequantizelinear"}}, - truncate_float_pass{ins_names, shape::half_type}, - optimize_module{{"quantizelinear", "dequantizelinear"}}}, - quant_tracer()); -} - -void quantize_bf16(program& prog, const std::vector& ins_names) -{ - run_passes(prog, - {normalize_ops{}, - optimize_module{{"quantizelinear", "dequantizelinear"}}, - truncate_float_pass{ins_names, shape::bf16_type}, - optimize_module{{"quantizelinear", "dequantizelinear"}}}, - quant_tracer()); -} - -void quantize_8bits(program& prog, - const target& t, - shape::type_t precision, - const std::vector& calibration, - const std::unordered_set& ins_names) -{ - // Run optimize_module() before converting to int8/fp8 to const eval and fold in FP32 to - // avoid loss of precision. - run_passes(prog, {normalize_ops{}, optimize_module{}}, quant_tracer()); - - std::shared_ptr>> quant_8bit_params = - std::make_shared>>(); - std::shared_ptr> max_abs_vals = std::make_shared>(); - std::map type_ranges = {{shape::type_t::int8_type, 127.0}, - {shape::type_t::fp8e4m3fnuz_type, 240.0}, - {shape::type_t::fp8e4m3fn_type, 448.0}}; - float quantized_range = type_ranges.at(precision); - auto calc_quant_params = [&](std::size_t ins_index, std::vector args) { - std::pair param_pair{64.0f, 0.0f}; - // scale and shift is need for only int8 type, and we do not - // consider shift, so set shift to 0 - std::vector vec_val; - argument arg = t.copy_from(args.front()); - arg.visit([&](auto output) { vec_val.assign(output.begin(), output.end()); }); - auto max_val = *std::max_element(vec_val.begin(), vec_val.end()); - auto min_val = *std::min_element(vec_val.begin(), vec_val.end()); - auto max_abs = std::max(std::fabs(max_val), std::fabs(min_val)); - max_abs_vals->at(ins_index) = std::max(max_abs_vals->at(ins_index), max_abs); - // if all values are 0, no need to do scaling - if(float_equal(max_abs_vals->at(ins_index), 0.0f)) - { - param_pair.first = 1.0f; - } - else - { - param_pair.first = quantized_range / max_abs_vals->at(ins_index); - } - quant_8bit_params->at(ins_index) = param_pair; - }; - - // pass to add capture argument op - std::size_t param_num = 0; - run_passes( - prog, {capture_arguments_pass{ins_names, calc_quant_params, ¶m_num}}, quant_tracer()); - quant_8bit_params->resize(param_num, std::pair(64.0f, 0.0f)); - max_abs_vals->resize(param_num, 0.0f); - - // use the calibration data to compute the quantization scale - auto capture_prog = prog; - capture_prog.compile(t); - - // use all calibration data to run the program to calculate the - // quantization scale and shift - for(auto&& arg : calibration) - { - parameter_map m; - for(auto&& x : capture_prog.get_parameter_shapes()) - { - if(arg.count(x.first) > 0) - { - assert(x.second == arg.at(x.first).get_shape()); - m[x.first] = t.copy_to(arg.at(x.first)); - } - else - { - m[x.first] = t.allocate(x.second); - } - } - capture_prog.eval(m); - } - - // print the quantization parameters in only the main module - if(enabled(MIGRAPHX_8BITS_QUANTIZATION_PARAMS{})) - { - for(std::size_t i = 0; i < quant_8bit_params->size(); ++i) - { - auto param = quant_8bit_params->at(i); - std::cout << "ins_index = " << i << ", scale = " << param.first - << ", shift = " << param.second << std::endl; - } - std::cout << std::endl; - } - - run_passes(prog, - {quantize_8bits_pass{precision, *quant_8bit_params}, dead_code_elimination{}}, - quant_tracer()); -} - -void quantize_int8(program& prog, - const target& t, - const std::vector& calibration, - const std::unordered_set& ins_names) -{ - std::unordered_set op_names = {"convolution", "dot"}; - if(op_names != ins_names) - { - MIGRAPHX_THROW("QUANTIZE_INT8: only support DOT and CONVOLUTION operation"); - } - quantize_8bits(prog, t, shape::int8_type, calibration, ins_names); -} - -void quantize_int4_weights(program& prog) -{ - run_passes(prog, {normalize_ops{}, optimize_module{}, quantize_int4_pass{}}, quant_tracer()); -} - -void quantize_fp8(program& prog, const target& t, const std::vector& calibration) -{ - std::unordered_set supported_ins_names; - auto* mm = prog.get_main_module(); - for(auto ins : iterator_for(*mm)) - { - if(ins->name() == "convert") - { - continue; - } - if(not starts_with(ins->name(), "@")) - { - supported_ins_names.insert(ins->name()); - } - } - quantize_8bits(prog, t, shape::fp8e4m3fn_type, calibration, supported_ins_names); -} -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/quantize_8bits.cpp b/docker/rocm/migraphx/quantize_8bits.cpp deleted file mode 100644 index 379ae94b6..000000000 --- a/docker/rocm/migraphx/quantize_8bits.cpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static std::vector& get_quantizable_type() -{ - static std::vector quantable_types = { - shape::float_type, shape::double_type, shape::half_type}; - return quantable_types; -} - -void quantize_8bits_pass::apply(module& m) const // NOLINT -{ - const auto& quantizable_types = get_quantizable_type(); - for(auto ins : iterator_for(m)) - { - if(ins->name() != "capture") - continue; - - auto op_val = ins->get_operator().to_value(); - assert(op_val.contains("ins_index")); - - auto param_index = op_val.at("ins_index").to(); - auto param = quant_params[param_index]; - - auto input = ins->inputs().front(); - auto s = input->get_shape(); - if(contains(quantizable_types, s.type()) and s.type() != precision) - { - auto zero_point = - m.add_literal(migraphx::literal{migraphx::shape{precision}, {param.second}}); - auto scale = m.add_literal(literal({s.type()}, {1.0f / param.first})); - const auto& lens = s.lens(); - scale = - m.insert_instruction(ins, make_op("multibroadcast", {{"out_lens", lens}}), scale); - zero_point = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", lens}}), zero_point); - auto q_in = - m.insert_instruction(ins, make_op("quantizelinear"), input, scale, zero_point); - auto dq_in = - m.insert_instruction(ins, make_op("dequantizelinear"), q_in, scale, zero_point); - m.replace_instruction(ins, dq_in); - } - } -} - -void capture_arguments_pass::apply(module& m) const // NOLINT -{ - assert(param_index != nullptr); - const auto& quantizable_types = get_quantizable_type(); - - for(auto ins : iterator_for(m)) - { - if((not contains(ins_names, ins->name())) or (ins->name() == "convert")) - { - continue; - } - - auto inputs = ins->inputs(); - std::vector new_args; - for(auto input : inputs) - { - if(contains(quantizable_types, input->get_shape().type())) - { - auto new_in = m.insert_instruction(ins, op::capture{(*param_index)++, f}, input); - new_args.push_back(new_in); - } - else - { - new_args.push_back(input); - } - } - m.replace_instruction(ins, ins->get_operator(), new_args); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/quantize_int4.cpp b/docker/rocm/migraphx/quantize_int4.cpp deleted file mode 100644 index 12b59d088..000000000 --- a/docker/rocm/migraphx/quantize_int4.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static void int4_quantize_module(module& m) -{ - std::vector int4_instrs{"dot", "convolution"}; - - for(auto ins : iterator_for(m)) - { - if(not(contains(int4_instrs, ins->name()))) - continue; - - if(ins->inputs().empty()) - continue; - - auto s = ins->get_shape(); - - auto mod_inputs = ins->module_inputs(); - - // Convert each of the inputs that are fp32 or fp16 to int4 - auto inputs = ins->inputs(); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](auto inp) { - auto sh = inp->get_shape(); - if(sh.broadcasted()) - return inp; - auto input_type = sh.type(); - if(input_type != shape::float_type and input_type != shape::half_type) - return inp; - auto lens = sh.lens(); - if(lens[lens.size() - 1] % 2) - return inp; // even sized dimensions to pack - - if(not inp->can_eval()) - return inp; - - std::vector val; - inp->eval().visit([&](auto in_data) { val.assign(in_data.begin(), in_data.end()); }); - - auto [min, max] = std::minmax_element(val.begin(), val.end()); - *min = *min > 0 ? 0 : *min; - *max = *max < 0 ? 0 : *max; - float fscale4 = (*max - *min) / 15; // INT4 range is [0-15] - int zp4 = float_equal(fscale4, 0) ? 0 : std::round(-*min / fscale4); - - auto scale = m.add_literal(literal({s.type()}, {fscale4})); - scale = - m.insert_instruction(ins, make_op("multibroadcast", {{"out_lens", lens}}), scale); - auto zp = m.add_literal(literal{{shape::uint8_type}, {zp4}}); - zp = m.insert_instruction(ins, make_op("multibroadcast", {{"out_lens", lens}}), zp); - auto q_in = m.insert_instruction(ins, make_op("quantizelinear"), inp, scale, zp); - - auto pk = m.insert_instruction(ins, make_op("pack_int4", {{"axis", -1}}), q_in); - auto unpk = m.insert_instruction(ins, make_op("unpack_int4", {{"axis", -1}}), pk); - - auto dq_scale = m.add_literal(literal({s.type()}, {fscale4})); - dq_scale = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", lens}}), dq_scale); - - auto dq_zp = m.add_literal(literal{{shape::uint8_type}, {zp4}}); - dq_zp = - m.insert_instruction(ins, make_op("multibroadcast", {{"out_lens", lens}}), dq_zp); - - return m.insert_instruction(ins, make_op("dequantizelinear"), unpk, dq_scale, dq_zp); - }); - - auto converted_ins = m.insert_instruction(ins, ins->get_operator(), inputs, mod_inputs); - m.replace_instruction(ins, converted_ins); - } -} - -void quantize_int4_pass::apply(module& m) const { int4_quantize_module(m); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/reduce_dims.cpp b/docker/rocm/migraphx/reduce_dims.cpp deleted file mode 100644 index 025f93184..000000000 --- a/docker/rocm/migraphx/reduce_dims.cpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -bool reduce_dim(std::vector& shapes, std::size_t n) -{ - std::vector new_lens; - for(const auto& s : shapes) - { - assert(n < s.lens().size()); - if((n + 1) >= s.lens().size()) - return false; - auto astride = s.strides()[n]; - auto alen = s.lens()[n]; - auto bstride = s.strides()[n + 1]; - auto blen = s.lens()[n + 1]; - - if(astride == bstride * blen or alen == 1) - new_lens.push_back(alen * blen); - } - if(new_lens.size() != shapes.size()) - return false; - std::size_t i = 0; - for(auto& s : shapes) - { - auto lens = s.lens(); - auto strides = s.strides(); - lens.erase(lens.begin() + n); - strides.erase(strides.begin() + n); - lens[n] = new_lens[i]; - s = shape{s.type(), lens, strides}; - i++; - } - return true; -} - -void reduce_dim1(std::vector& shapes) -{ - if(std::any_of(shapes.begin(), shapes.end(), [&](const auto& s) { - return s.lens().size() < 2 or s.lens().back() != 1; - })) - return; - for(auto& s : shapes) - { - auto lens = s.lens(); - auto strides = s.strides(); - lens.pop_back(); - strides.pop_back(); - s = shape{s.type(), lens, strides}; - } -} - -std::size_t reduce_dim_all(std::vector& shapes, std::size_t n) -{ - while(reduce_dim(shapes, n) and n < shapes.size()) - { - (void)n; - } - return n + 1; -} -void reduce_dim_all(std::vector& shapes) -{ - std::size_t n = 0; - while(n < shapes.front().lens().size() - 1) - n = reduce_dim_all(shapes, n); - reduce_dim1(shapes); -} - -std::vector base_lens(const std::vector& shapes) -{ - return std::accumulate( - shapes.begin() + 1, shapes.end(), shapes.front().lens(), [](auto&& lens, auto&& s) { - std::vector result; - const auto* x = &s.lens(); - const auto* y = &lens; - if(x->size() > y->size()) - std::swap(x, y); - std::transform( - x->begin(), x->end(), y->begin(), std::back_inserter(result), [&](auto a, auto b) { - return std::max(a, b); - }); - return result; - }); -} - -shape mask_shape(const shape& s, const std::vector& lens) -{ - assert(s.lens().size() == lens.size()); - std::vector rstrides(lens.size()); - std::size_t stride = 1; - for(std::size_t i = lens.size() - 1; i < lens.size(); i--) - { - if(lens[i] == s.lens()[i]) - { - rstrides[i] = stride; - stride *= lens[i]; - } - else if(lens[i] != 1 and s.lens()[i] != 1) - { - return shape{}; - } - } - return shape{s.type(), lens, rstrides}; -} - -std::vector reduce_dims(const std::vector& shapes) -{ - if(shapes.empty()) - return {}; - auto result = shapes; - auto base = base_lens(shapes); - for(auto&& s : shapes) - { - if(s.lens().size() != base.size()) - return shapes; - if(s.lens() == base) - continue; - auto mshape = mask_shape(s, base); - if(mshape.lens().size() != base.size()) - return shapes; - result.push_back(mshape); - } - reduce_dim_all(result); - result.erase(result.begin() + shapes.size(), result.end()); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/register_op.cpp b/docker/rocm/migraphx/register_op.cpp deleted file mode 100644 index c7b5b4803..000000000 --- a/docker/rocm/migraphx/register_op.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -std::unordered_map& op_map() -{ - static std::unordered_map m; // NOLINT - return m; -} - -void register_op_init() { (void)op_map(); } - -void register_op(const operation& op) { op_map()[op.name()] = op; } - -void unregister_op(const std::string& op_name) -{ - assert(op_map().count(op_name)); - op_map().erase(op_name); -} - -operation load_op(const std::string& name) -{ - return at(op_map(), name, "Operator not found: " + name); -} - -bool has_op(const std::string& name) { return op_map().count(name) == 1; } - -std::vector get_operators() -{ - std::vector result; - std::transform(op_map().begin(), op_map().end(), std::back_inserter(result), [&](auto&& p) { - return p.first; - }); - std::sort(result.begin(), result.end()); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/register_target.cpp b/docker/rocm/migraphx/register_target.cpp deleted file mode 100644 index 241b080c0..000000000 --- a/docker/rocm/migraphx/register_target.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void store_target_lib(const dynamic_loader& lib) -{ - static std::vector target_loader; - target_loader.emplace_back(lib); -} - -std::unordered_map& target_map() -{ - static std::unordered_map m; // NOLINT - return m; -} - -void register_target_init() { (void)target_map(); } - -void unregister_target(const std::string& name) -{ - assert(target_map().count(name)); - target_map().erase(name); -} - -void register_target(const target& t) { target_map()[t.name()] = t; } - -target make_target(const std::string& name) -{ - if(not contains(target_map(), name)) - { - std::string so_major_version = "." + std::to_string(MIGRAPHX_SO_MAJOR_VERSION); - auto target_name = make_shared_object_filename("migraphx_" + name); - - // Try to load library with so_major_version appended to the name. - // If library with so_major_version name is not found, - // try loading the library without the so_major_version name appended. - // For example, if "libmigraphx_ref.so.2010000" is not found, - // try loading "libmigraphx_ref.so". - try - { - // Default to loading shared libraries with - // so_major_version appended. - store_target_lib(dynamic_loader(target_name + so_major_version)); - } - catch(...) - { - // Load the library without the so_major_version in the name. - store_target_lib(dynamic_loader(target_name)); - } - } - const auto it = target_map().find(name); - if(it == target_map().end()) - { - MIGRAPHX_THROW("Requested target '" + name + "' is not loaded or not supported"); - } - return it->second; -} - -std::vector get_targets() -{ - std::vector result; - std::transform(target_map().begin(), - target_map().end(), - std::back_inserter(result), - [&](auto&& p) { return p.first; }); - std::sort(result.begin(), result.end()); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/replace_allocate.cpp b/docker/rocm/migraphx/replace_allocate.cpp deleted file mode 100644 index 79e6bc763..000000000 --- a/docker/rocm/migraphx/replace_allocate.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -std::unordered_map create_output_names(const module& mod) -{ - std::unordered_map mod_output_names{}; - auto last = std::prev(mod.end()); - if(last->name() == "@return") - { - const auto& prog_outputs = last->inputs(); - std::vector outputs_alias(prog_outputs.size()); - - std::transform(prog_outputs.begin(), - prog_outputs.end(), - outputs_alias.begin(), - [](const auto& i) { return instruction::get_output_alias(i); }); - - std::size_t index = 0; - for(auto ins : outputs_alias) - { - mod_output_names[ins] = mod.name() + ":#output_" + std::to_string(index++); - } - } - else - { - auto ins = instruction::get_output_alias(last); - mod_output_names[ins] = "output"; - } - return mod_output_names; -} - -void insert_submod_allocations(instruction_ref ins, module& mod, const allocation_model& model) -{ - std::vector inputs = ins->inputs(); - std::vector mod_args = ins->module_inputs(); - - std::map name_shapes; - for(const auto& smod : mod_args) - { - auto ps = smod->get_parameter_shapes(); - name_shapes.insert(ps.begin(), ps.end()); - } - - for(const auto& pn : name_shapes) - { - const auto& s = pn.second; - instruction_ref output{}; - output = mod.insert_instruction(ins, model.allocate(s)); - inputs.push_back(output); - } - - mod.replace_instruction(ins, ins->get_operator(), inputs, mod_args); -} - -void replace_allocate::apply(module_pass_manager& mpm) const -{ - module& m = mpm.get_module(); - auto mod_output_names = create_output_names(m); - bool root_offload_copy = (*mpm.get_root_module() == m) ? this->offload_copy : false; - for(auto ins : iterator_for(m)) - { - auto op = ins->get_operator(); - auto op_name = op.name(); - - // check if allocations from submodules need to be inserted - // for now, only the "if" operator is affected - if(op_name == "if") - { - insert_submod_allocations(ins, m, model); - continue; - } - if(op_name != "allocate") - continue; - - auto s = ins->get_shape(); - if(not root_offload_copy and model.needs_out_params() and contains(mod_output_names, ins)) - { - auto out_param = m.add_parameter(mod_output_names[ins], s); - m.replace_instruction(ins, out_param); - } - else - { - m.replace_instruction(ins, - make_op(model.name(), migraphx::value{{"shape", to_value(s)}})); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/rewrite_gelu.cpp b/docker/rocm/migraphx/rewrite_gelu.cpp deleted file mode 100644 index 2d87e4d05..000000000 --- a/docker/rocm/migraphx/rewrite_gelu.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * The replacement approximation is equivalent to: - * GELU(x) ~= 0.5 * x * ( 1 + tanh( sqrt(2/M_PI) * (x + 0.044715 * x^3))) - * You can rearrange to the form used in this by recognizing that - * 1 + tanh(x) = (2) / (1 + exp(-2 * x)). - * The fitting constant 0.044715 is from - * A. Choudhury, ‘A simple approximation to the area under standard normal curve’, Mathematics and - * Statistics, vol. 2, no. 3, pp. 147–149, 2014. - */ -void replace_with_tanh_exp_gelu(module& m, const match::matcher_result& r) -{ - auto ins = r.result; - auto x = r.instructions["x"]; - double const0 = -2. * sqrt(M_2_PI); - double const1 = 0.044715 * const0; - auto lit0 = m.add_literal(literal{shape{x->get_shape().type()}, {const0}}); - auto lit1 = m.add_literal(literal{shape{x->get_shape().type()}, {const1}}); - auto one = m.add_literal(literal{shape{x->get_shape().type()}, {1.0}}); - auto xb = insert_common_op(m, ins, make_op("mul"), {x, lit1}); - auto a = m.insert_instruction(ins, make_op("mul"), x, xb); - auto b = insert_common_op(m, ins, make_op("add"), {a, lit0}); - auto u = m.insert_instruction(ins, make_op("mul"), x, b); - auto emu = m.insert_instruction(ins, make_op("exp"), u); - auto c = insert_common_op(m, ins, make_op("add"), {one, emu}); - auto y = m.insert_instruction(ins, make_op("div"), x, c); - m.replace_instruction(ins, y); -} - -/** - * Finds erfGELU blocks using the Gaussian distribution and replaces them with the tanh_exp - * approximation if the data type is fp16. TODO consider also for fp8 datatype. - */ -struct find_gelu_erf -{ - auto matcher() const { return match::any_of(match::gelu_erf(), match::gelu_tanh()); } - - void apply(module& m, const match::matcher_result& r) const - { - auto x = r.instructions["x"]; - auto input_type = x->get_shape().type(); - std::set convert_types = {migraphx::shape::half_type}; - if(not contains(convert_types, input_type)) - return; - - replace_with_tanh_exp_gelu(m, r); - } -}; - -/** - * Find tanhGELU blocks and replace them with a rearranged version that is less likely to overflow - * and is more performant. - */ -struct find_tanh_fast_gelu -{ - auto matcher() const { return match::gelu_tanh(); } - - void apply(module& m, const match::matcher_result& r) const - { - replace_with_tanh_exp_gelu(m, r); - } -}; - -void rewrite_gelu::apply(module& m) const -{ - if(fast_math) - { - match::find_matches(m, find_gelu_erf{}); - } - else - { - match::find_matches(m, find_tanh_fast_gelu{}); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/rewrite_low_precision.cpp b/docker/rocm/migraphx/rewrite_low_precision.cpp deleted file mode 100644 index b091fa01e..000000000 --- a/docker/rocm/migraphx/rewrite_low_precision.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct find_pow2_div -{ - - auto pow2() const - { - auto pow2 = match::name("pow")(match::arg(0)(match::any().bind("x")), - match::arg(1)(match::has_value(2.0f))); - auto x_square = - match::name("mul")(match::same_inputs(), match::arg(0)(match::any().bind("x"))); - return match::any_of(pow2, x_square); - } - - auto matcher() const - { - return match::name("div")(match::arg(0)(pow2()), - match::arg(1)(match::is_constant().bind("n"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto n = r.instructions["n"]; - auto x = r.instructions["x"]; - - if(x->get_shape().type() != migraphx::shape::half_type) - return; - auto x_div_n = m.insert_instruction(ins, make_op("div"), {x, n}); - auto mul = m.insert_instruction(ins, make_op("mul"), {x_div_n, x}); - m.replace_instruction(ins, mul); - } -}; - -void rewrite_low_precision::apply(module& m) const { match::find_matches(m, find_pow2_div{}); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/rewrite_pooling.cpp b/docker/rocm/migraphx/rewrite_pooling.cpp deleted file mode 100644 index 6100043e8..000000000 --- a/docker/rocm/migraphx/rewrite_pooling.cpp +++ /dev/null @@ -1,185 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static void replace_with_reduce(module& m, instruction_ref ins) -{ - auto&& s = ins->inputs().front()->get_shape(); - auto&& op = any_cast(ins->get_operator()); - auto lens = s.lens(); - std::vector axes(lens.size() - 2); - std::iota(axes.begin(), axes.end(), 2); - - // average pooling - if(op.mode == op::pooling_mode::average) - { - m.replace_instruction(ins, make_op("reduce_mean", {{"axes", axes}}), ins->inputs()); - } - // max pooling - else - { - m.replace_instruction(ins, make_op("reduce_max", {{"axes", axes}}), ins->inputs()); - } -} - -static void replace_dilations_with_gather_pooling(module& m, instruction_ref ins) -{ - // TODO remove this when MIOpen supports dilated pooling - auto&& s = ins->inputs().front()->get_shape(); - auto&& op = any_cast(ins->get_operator()); - // Ignore N, C axes - std::vector dims = {s.lens().cbegin() + 2, s.lens().cend()}; - - bool default_padding = - std::all_of(op.padding.cbegin(), op.padding.cend(), [](auto i) { return i == 0; }); - - if(not default_padding) - { - for(size_t idx{0}; idx < op.padding.size(); ++idx) - { - // We need to pad both ends - dims[idx] += op.padding.at(idx) * 2; - } - } - std::vector kernels = op.lengths; - std::vector strides = op.stride; - std::vector dilations = op.dilations; - - std::vector> axis_indices; - axis_indices.resize(dims.size()); - - for(auto idx{0}; idx < dims.size(); ++idx) - { - // Only consider if iw fits into the window - for(size_t stride{0}; stride < dims.at(idx) - dilations.at(idx) * (kernels.at(idx) - 1); - stride += strides.at(idx)) - { - for(size_t step{0}; step < kernels.at(idx); ++step) - { - axis_indices.at(idx).push_back(stride + dilations.at(idx) * step); - } - } - } - - auto elements = ins->inputs().front(); - if(not default_padding) - { - // Pad supports asym, we need to provide both ends - std::vector padding(2 * s.lens().size(), 0); - // Format will be e.g {N, C, P1, P2, N, C, P1, P2} - for(size_t idx{0}; idx < op.padding.size(); ++idx) - { - // Ignore N, C axes - padding.at(2 + idx) = op.padding.at(idx); - padding.at(2 + idx + s.lens().size()) = op.padding.at(idx); - } - - // Default value needed for Max pooling - elements = m.insert_instruction( - ins, - make_op("pad", {{"pads", padding}, {"value", std::numeric_limits::lowest()}}), - elements); - } - - for(auto idx{0}; idx < axis_indices.size(); ++idx) - { - migraphx::shape s_indices{migraphx::shape::int32_type, {axis_indices.at(idx).size()}}; - auto indices = m.add_literal(migraphx::literal{s_indices, axis_indices.at(idx)}); - elements = m.insert_instruction( - ins, make_op("gather", {{"axis", idx + 2 /*ignore N,C*/}}), elements, indices); - } - - // Ignore padding - std::vector new_padding(kernels.size(), 0); - // The kernel window elements are places next to each other. E.g. {x1, y1, x2, y2, ...} - // We need to skip them to not overlap - std::vector new_strides(kernels); - // Ignore dilations - std::vector new_dilations(kernels.size(), 1); - m.replace_instruction(ins, - make_op("pooling", - {{"mode", op.mode}, - {"padding", new_padding}, - {"stride", new_strides}, - {"lengths", kernels}, - {"dilations", new_dilations}}), - elements); -} - -void rewrite_pooling::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "pooling") - continue; - if(ins->inputs().empty()) - continue; - auto&& s = ins->inputs().front()->get_shape(); - auto&& op = any_cast(ins->get_operator()); - bool same_kernel_as_shape = std::equal( - s.lens().cbegin() + 2, s.lens().cend(), op.lengths.cbegin(), op.lengths.cend()); - bool default_strides = - std::all_of(op.stride.cbegin(), op.stride.cend(), [](auto i) { return i == 1; }); - bool default_padding = - std::all_of(op.padding.cbegin(), op.padding.cend(), [](auto i) { return i == 0; }); - bool default_dilations = - std::all_of(op.dilations.cbegin(), op.dilations.cend(), [](auto i) { return i == 1; }); - if(same_kernel_as_shape and default_strides and default_padding and default_dilations) - { - replace_with_reduce(m, ins); - } - else if(not default_dilations) - { - // Dilated AvgPool with padding is not supported - if(not default_padding and op.mode == op::pooling_mode::average) - { - continue; - } - auto size = - std::accumulate(s.lens().cbegin(), s.lens().cend(), 1, std::multiplies()); - // Can't handle too much size because of literal size - if(size > 100000) - { - continue; - } - - replace_dilations_with_gather_pooling(m, ins); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/rewrite_quantization.cpp b/docker/rocm/migraphx/rewrite_quantization.cpp deleted file mode 100644 index 90e98ba89..000000000 --- a/docker/rocm/migraphx/rewrite_quantization.cpp +++ /dev/null @@ -1,124 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_CK_WORKAROUNDS); - -void apply_quantizelinear(module& m, instruction_ref ins) -{ - assert(ins->name() == "quantizelinear"); - auto x = ins->inputs()[0]; - auto y_scale = ins->inputs()[1]; - - if(x->get_shape().type() != y_scale->get_shape().type()) - { - x = m.insert_instruction( - ins, make_op("convert", {{"target_type", y_scale->get_shape().type()}}), x); - } - auto div = m.insert_instruction(ins, make_op("div"), x, y_scale); - auto add_zero_point = m.insert_instruction(ins, make_op("nearbyint"), div); - - if(ins->inputs().size() == 3) - { - auto zero_point = - m.insert_instruction(ins, - make_op("convert", {{"target_type", y_scale->get_shape().type()}}), - ins->inputs()[2]); - add_zero_point = m.insert_instruction(ins, make_op("add"), add_zero_point, zero_point); - } - - double max_quant = 0; - double min_quant = 0; - ins->get_shape().visit_type([&](auto qt) { - max_quant = qt.max(); - min_quant = qt.min(); - }); - auto s = add_zero_point->get_shape(); - instruction_ref min_arg; - instruction_ref max_arg; - - if(enabled(MIGRAPHX_ENABLE_CK_WORKAROUNDS{})) - { - std::vector min_data(s.elements(), min_quant); - std::vector max_data(s.elements(), max_quant); - min_arg = m.add_literal(literal(s, min_data)); - max_arg = m.add_literal(literal(s, max_data)); - } - else - { - min_arg = m.add_literal(literal{shape{s.type()}, {min_quant}}); - max_arg = m.add_literal(literal{shape{s.type()}, {max_quant}}); - } - auto saturate = insert_common_op(m, ins, make_op("clip"), {add_zero_point, min_arg, max_arg}); - m.replace_instruction( - ins, make_op("convert", {{"target_type", ins->get_shape().type()}}), saturate); -} - -void apply_dequantizelinear(module& m, instruction_ref ins) -{ - assert(ins->name() == "dequantizelinear"); - auto x_scale = ins->inputs()[1]; - auto x = m.insert_instruction( - ins, make_op("convert", {{"target_type", x_scale->get_shape().type()}}), ins->inputs()[0]); - - if(ins->inputs().size() == 3) - { - auto x_zero_point = - m.insert_instruction(ins, - make_op("convert", {{"target_type", x_scale->get_shape().type()}}), - ins->inputs()[2]); - x = m.insert_instruction(ins, make_op("sub"), x, x_zero_point); - } - - m.replace_instruction(ins, make_op("mul"), x, x_scale); -} - -void rewrite_quantization::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() == "quantizelinear") - { - apply_quantizelinear(m, ins); - } - - else if(ins->name() == "dequantizelinear") - { - apply_dequantizelinear(m, ins); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/rewrite_reduce.cpp b/docker/rocm/migraphx/rewrite_reduce.cpp deleted file mode 100644 index 239d255c3..000000000 --- a/docker/rocm/migraphx/rewrite_reduce.cpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace { -struct find_softmax -{ - auto matcher() const { return match::name("softmax"); } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto op = ins->get_operator().to_value(); - auto axis = op["axis"].to(); - - auto input = ins->inputs().front(); - auto max = m.insert_instruction(ins, make_op("reduce_max", {{"axes", {axis}}}), input); - auto maxb = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", input->get_shape().lens()}}), max); - auto sub = m.insert_instruction(ins, make_op("sub"), input, maxb); - auto exp = m.insert_instruction(ins, make_op("exp"), sub); - auto sum = m.insert_instruction(ins, make_op("reduce_sum", {{"axes", {axis}}}), exp); - auto sumb = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", input->get_shape().lens()}}), sum); - m.replace_instruction(ins, make_op("div"), exp, sumb); - } -}; - -struct find_reduce_mean_variance -{ - auto matcher() const - { - auto reduce_mean = match::name("reduce_mean"); - auto skip_broadcasts_mean = match::skip_broadcasts(reduce_mean.bind("mean")); - auto x_minus_mean = match::name("sub")(match::arg(0)(match::any().bind("x")), - match::arg(1)(skip_broadcasts_mean)); - auto pow_x_minus_mean = - match::name("pow")(match::arg(0)(x_minus_mean), match::arg(1)(match::has_value(2.0f))); - auto mul_x_minus_mean = - match::name("mul")(match::arg(0)(x_minus_mean), match::arg(1)(x_minus_mean)); - auto sqdiff = match::name("sqdiff")( - match::either_arg(0, 1)(match::any().bind("x"), skip_broadcasts_mean)); - return reduce_mean( - match::arg(0)(match::any_of(pow_x_minus_mean, mul_x_minus_mean, sqdiff))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - auto mean = r.instructions["mean"]; - - if(ins->get_operator() != mean->get_operator()) - return; - - if(mean->inputs().front() != x_ins) - return; - - auto x2 = m.insert_instruction(ins, make_op("mul"), x_ins, x_ins); - auto mean_x2 = m.insert_instruction(ins, mean->get_operator(), x2); - auto mean_x_2 = m.insert_instruction(ins, make_op("mul"), mean, mean); - m.replace_instruction(ins, make_op("sub"), mean_x2, mean_x_2); - } -}; - -struct find_reduce_mean -{ - auto matcher() const { return match::name("reduce_mean"); } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto op = ins->get_operator().to_value(); - auto axes = op["axes"].to_vector(); - auto input = ins->inputs().front(); - - bool is_integral = false; - double max_n = 0; - std::size_t size = 0; - input->get_shape().visit_type([&](auto t) { - is_integral = t.is_integral(); - max_n = t.max(); - size = t.size(); - }); - - auto n = input->get_shape().elements() / ins->get_shape().elements(); - - // Convert accumulator to float if <= 8bit type or if < 3 bytes and n >= max_n /4 - if(size == 1 or (n >= max_n / 4 and size < 3)) - { - shape::type_t t = is_integral ? shape::int32_type : shape::float_type; - input = m.insert_instruction(ins, make_op("convert", {{"target_type", t}}), input); - } - - auto n_literal = m.add_literal(literal{{input->get_shape().type(), {1}}, {n}}); - if(is_integral) - { - auto reduce_sum = - m.insert_instruction(ins, make_op("reduce_sum", {{"axes", axes}}), input); - auto div = insert_common_op(m, ins, make_op("div"), {reduce_sum, n_literal}); - m.replace_instruction( - ins, make_op("convert", {{"target_type", ins->get_shape().type()}}), div); - } - else - { - auto new_input = insert_common_op(m, ins, make_op("div"), {input, n_literal}); - auto reduce_sum = - m.insert_instruction(ins, make_op("reduce_sum", {{"axes", axes}}), new_input); - m.replace_instruction( - ins, make_op("convert", {{"target_type", ins->get_shape().type()}}), reduce_sum); - } - } -}; - -} // namespace - -void rewrite_reduce::apply(module& m) const -{ - match::find_matches(m, find_softmax{}, find_reduce_mean_variance{}); - match::find_matches(m, find_reduce_mean{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/rewrite_rnn.cpp b/docker/rocm/migraphx/rewrite_rnn.cpp deleted file mode 100644 index b16835851..000000000 --- a/docker/rocm/migraphx/rewrite_rnn.cpp +++ /dev/null @@ -1,1443 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void rewrite_rnn::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() == "rnn") - { - apply_vanilla_rnn(m, ins); - } - else if(ins->name() == "gru") - { - apply_gru(m, ins); - } - else if(ins->name() == "lstm") - { - apply_lstm(m, ins); - } - } -} - -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -void rewrite_rnn::apply_vanilla_rnn(module& m, instruction_ref ins) const -{ - assert(ins->name() == "rnn"); - // could be 3 to 6 inputs, but the parse_rnn function will - // append undefined operators to make 6 arguments when parsing - // an onnx file. Another case is user can have num of arguments - // when writing their module. - auto args = ins->inputs(); - - shape seq_shape = args[0]->get_shape(); - std::size_t hidden_size = args[1]->get_shape().lens()[1]; - std::size_t batch_size = seq_shape.lens()[1]; - shape::type_t type = seq_shape.type(); - migraphx::shape ih_shape{type, {1, batch_size, hidden_size}}; - std::vector data(ih_shape.elements(), 0); - - auto actv_funcs = vanilla_rnn_actv_funcs(ins); - auto rnn_op = any_cast(ins->get_operator()); - op::rnn_direction dirct = rnn_op.direction; - - // process sequence length - instruction_ref seq_lens = m.end(); - if((args.size() >= 5) and not args[4]->is_undefined()) - { - seq_lens = args[4]; - } - - bool variable_seq_len = is_variable_seq_lens(m, seq_lens); - - instruction_ref last_output{}; - if(dirct == op::rnn_direction::bidirectional) - { - // input weight matrix - auto w_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[1]); - auto w_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[1]); - - // hidden state weight matrix - auto r_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[2]); - auto r_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[2]); - - // process bias - instruction_ref bias_forward = m.end(); - instruction_ref bias_reverse = m.end(); - if(args.size() >= 4 and not args[3]->is_undefined()) - { - bias_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[3]); - bias_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[3]); - } - - // process intial hidden state, it could be the 6th argument - // or the 5th one (if the sequence len argument is ignored) - instruction_ref ih_forward{}; - instruction_ref ih_reverse{}; - if(args.size() == 6 and not args[5]->is_undefined()) - { - ih_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[5]); - ih_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[5]); - } - else - { - ih_forward = m.add_literal(migraphx::literal{ih_shape, data}); - ih_reverse = m.add_literal(migraphx::literal{ih_shape, data}); - } - - auto ret_forward = - vanilla_rnn_cell(true, - m, - ins, - {args[0], w_forward, r_forward, bias_forward, seq_lens, ih_forward}, - actv_funcs.at(0)); - - if(variable_seq_len) - { - args[0] = - m.insert_instruction(ins, make_op("rnn_var_sl_shift_sequence"), args[0], seq_lens); - } - - auto ret_reverse = - vanilla_rnn_cell(false, - m, - ins, - {args[0], w_reverse, r_reverse, bias_reverse, seq_lens, ih_reverse}, - actv_funcs.at(1)); - - auto concat_output = m.insert_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[1], ret_reverse[1]); - last_output = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), concat_output); - - // The following logic is to ensure the last instruction rewritten from - // rnn operator is a concat instruction - // sequence len is 1 - if(ret_forward[0] == m.end()) - { - m.replace_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[1], ret_reverse[1]); - } - else - { - ret_forward[0] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_forward[0], ret_forward[1]); - ret_reverse[0] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_reverse[1], ret_reverse[0]); - m.replace_instruction( - ins, make_op("concat", {{"axis", 1}}), {ret_forward[0], ret_reverse[0]}); - } - } - else - { - bool is_forward = (dirct == op::rnn_direction::forward); - // input weight matrix - auto w = args[1]; - - // hidden state weight matrix - auto r = args[2]; - - // process bias and initial hidden state - instruction_ref bias = m.end(); - if(args.size() >= 4 and not args[3]->is_undefined()) - { - bias = args[3]; - } - - // process intial hidden state - instruction_ref ih; - if(args.size() == 6 and not args[5]->is_undefined()) - { - ih = args[5]; - } - else - { - ih = m.add_literal(migraphx::literal{ih_shape, data}); - } - - if(not is_forward and variable_seq_len) - { - args[0] = - m.insert_instruction(ins, make_op("rnn_var_sl_shift_sequence"), args[0], seq_lens); - } - - auto ret = vanilla_rnn_cell( - is_forward, m, ins, {args[0], w, r, bias, seq_lens, ih}, actv_funcs.at(0)); - last_output = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ret[1]); - - // following logic is to ensure the last instruction is a - // concat instruction - // sequence len is 1 - if(ret[0] == m.end()) - { - m.replace_instruction(ins, make_op("concat", {{"axis", 0}}), ret[1]); - } - else - { - auto concat_arg0 = is_forward ? ret[0] : ret[1]; - auto concat_arg1 = is_forward ? ret[1] : ret[0]; - m.replace_instruction(ins, make_op("concat", {{"axis", 0}}), concat_arg0, concat_arg1); - } - } - - // in case of all sequences are of the same lengths and shorter than the - // max sequence length, need to pad 0's at the end for output hidden states - ins = pad_hidden_states(m, args[0], seq_lens, ins); - replace_last_hs_output(m, ins, seq_lens, last_output, dirct); -} - -std::vector rewrite_rnn::vanilla_rnn_cell(bool is_forward, - module& m, - instruction_ref ins, - std::vector inputs, - const operation& actv_func) const -{ - assert(inputs.size() == 6); - auto seq = inputs.at(0); - auto w = inputs.at(1); - auto r = inputs.at(2); - auto bias = inputs.at(3); - auto seq_lens = inputs.at(4); - auto ih = inputs.at(5); - - // squeeze and transpose w - std::vector perm{1, 0}; - auto sw = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), w); - auto tran_sw = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), sw); - - // squeeze and transpose r - auto sr = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), r); - auto tran_sr = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), sr); - - // initial hidden state - auto sih = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ih); - auto sih_lens = sih->get_shape().lens(); - - // bias - instruction_ref bb{}; - if(bias != m.end()) - { - long hs = r->get_shape().lens()[2]; - auto sbias = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), bias); - auto wb = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {hs}}}), sbias); - auto rb = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {hs}}, {"ends", {2 * hs}}}), sbias); - auto wrb = m.insert_instruction(ins, make_op("add"), wb, rb); - bb = m.insert_instruction( - ins, make_op("broadcast", {{"axis", 1}, {"out_lens", sih_lens}}), wrb); - } - - instruction_ref hidden_out = m.end(); - instruction_ref last_out{}; - last_out = m.insert_instruction(ins, make_op("unsqueeze", {{"axes", {0, 1}}}), sih); - long seq_len = get_seq_len(m, seq, seq_lens); - for(long i = 0; i < seq_len; i++) - { - long seq_index = is_forward ? i : (seq_len - 1 - i); - auto xt = m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {seq_index}}, {"ends", {seq_index + 1}}}), - seq); - auto cont_xt = m.insert_instruction(ins, make_op("contiguous"), xt); - xt = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), cont_xt); - auto xt_wi = m.insert_instruction(ins, make_op("dot"), xt, tran_sw); - auto ht_ri = m.insert_instruction(ins, make_op("dot"), sih, tran_sr); - if(bias != m.end()) - { - xt_wi = m.insert_instruction(ins, make_op("add"), xt_wi, bb); - } - auto xt_ht = m.insert_instruction(ins, make_op("add"), xt_wi, ht_ri); - - // apply activation function - auto ht = m.insert_instruction(ins, actv_func, xt_ht); - sih = ht; - - // add the dimensions of sequence length (axis 0 for sequence length, - // axis 1 for num_directions - last_out = m.insert_instruction(ins, make_op("unsqueeze", {{"axes", {0, 1}}}), ht); - - // concatenation for the last last_out is performed in the apply() - // function to ensure the last instruction is concat, then we have - // output inserted - if(i < seq_len - 1) - { - if(is_forward) - { - hidden_out = (seq_index == 0) - ? last_out - : m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), hidden_out, last_out); - } - else - { - hidden_out = (seq_index == seq_len - 1) - ? last_out - : m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), last_out, hidden_out); - } - } - } - - return {hidden_out, last_out}; -} - -std::vector rewrite_rnn::vanilla_rnn_actv_funcs(instruction_ref ins) const -{ - auto rnn_op = any_cast(ins->get_operator()); - // could be 3 to 6 inputs, but the parse_gru function will - // append undefined operators to make 6 arguments when parsing - // an onnx file. Another case is user can have any num of arguments - // when writing their program. - if(rnn_op.direction == op::rnn_direction::bidirectional) - { - if(rnn_op.actv_funcs.empty()) - { - // default is tanh - return {make_op("tanh"), make_op("tanh")}; - } - else if(rnn_op.actv_funcs.size() == 1) - { - return {rnn_op.actv_funcs.at(0), rnn_op.actv_funcs.at(0)}; - } - else - { - return rnn_op.actv_funcs; - } - } - else - { - if(rnn_op.actv_funcs.empty()) - { - // default is tanh - return {make_op("tanh")}; - } - else - { - return rnn_op.actv_funcs; - } - } -} - -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -void rewrite_rnn::apply_gru(module& m, instruction_ref ins) const -{ - assert(ins->name() == "gru"); - const auto actv_funcs = gru_actv_funcs(ins); - // could be 3 to 6 inputs, but the parse_gru function will - // append undefined operators to make 6 arguments when parsing - // an onnx file. Another case is user can have num of arguments - // when writing their program. - auto args = ins->inputs(); - - shape seq_shape = args[0]->get_shape(); - std::size_t hidden_size = args[2]->get_shape().lens()[2]; - std::size_t batch_size = seq_shape.lens()[1]; - shape::type_t type = seq_shape.type(); - migraphx::shape ih_shape{type, {1, batch_size, hidden_size}}; - std::vector data(ih_shape.elements(), 0.0); - - auto gru_op = any_cast(ins->get_operator()); - op::rnn_direction dirct = gru_op.direction; - - // process sequence length - instruction_ref seq_lens = m.end(); - if((args.size() >= 5) and not args[4]->is_undefined()) - { - seq_lens = args[4]; - } - - bool variable_seq_len = is_variable_seq_lens(m, seq_lens); - - instruction_ref last_output{}; - if(dirct == op::rnn_direction::bidirectional) - { - // w weight matrix - auto w_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[1]); - auto w_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[1]); - - // r weight matrix - auto r_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[2]); - auto r_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[2]); - - // bias - instruction_ref bias_forward = m.end(); - instruction_ref bias_reverse = m.end(); - if(args.size() >= 4 and not args[3]->is_undefined()) - { - bias_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[3]); - bias_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[3]); - } - - // intial hidden state - instruction_ref ih_forward{}; - instruction_ref ih_reverse{}; - if(args.size() == 6 and not args[5]->is_undefined()) - { - ih_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[5]); - ih_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[5]); - } - else - { - ih_forward = m.add_literal(migraphx::literal{ih_shape, data}); - ih_reverse = m.add_literal(migraphx::literal{ih_shape, data}); - } - - auto ret_forward = - gru_cell(true, - m, - ins, - {args[0], w_forward, r_forward, bias_forward, seq_lens, ih_forward}, - gru_op.linear_before_reset, - actv_funcs.at(0), - actv_funcs.at(1)); - - if(variable_seq_len) - { - args[0] = - m.insert_instruction(ins, make_op("rnn_var_sl_shift_sequence"), args[0], seq_lens); - } - - auto ret_reverse = - gru_cell(false, - m, - ins, - {args[0], w_reverse, r_reverse, bias_reverse, seq_lens, ih_reverse}, - gru_op.linear_before_reset, - actv_funcs.at(2), - actv_funcs.at(3)); - - auto concat_output = m.insert_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[1], ret_reverse[1]); - last_output = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), concat_output); - - // The following logic is to ensure the last instruction rewritten - // from gru operator is a concat - if(ret_forward[0] == m.end()) - { - m.replace_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[1], ret_reverse[1]); - } - else - { - ret_forward[0] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_forward[0], ret_forward[1]); - ret_reverse[0] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_reverse[1], ret_reverse[0]); - m.replace_instruction( - ins, make_op("concat", {{"axis", 1}}), {ret_forward[0], ret_reverse[0]}); - } - } - else - { - bool is_forward = (dirct == op::rnn_direction::forward); - // weight matrix - auto w = args[1]; - auto r = args[2]; - - // bias - instruction_ref bias = m.end(); - if(args.size() >= 4 and not args[3]->is_undefined()) - { - bias = args[3]; - } - - // intial hidden state - instruction_ref ih{}; - if(args.size() == 6 and not args[5]->is_undefined()) - { - ih = args[5]; - } - else - { - ih = m.add_literal(migraphx::literal{ih_shape, data}); - } - - if(not is_forward and variable_seq_len) - { - args[0] = - m.insert_instruction(ins, make_op("rnn_var_sl_shift_sequence"), args[0], seq_lens); - } - - auto ret = gru_cell(is_forward, - m, - ins, - {args[0], w, r, bias, seq_lens, ih}, - gru_op.linear_before_reset, - actv_funcs.at(0), - actv_funcs.at(1)); - - last_output = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ret[1]); - - if(ret[0] == m.end()) - { - m.replace_instruction(ins, make_op("concat", {{"axis", 0}}), ret[1]); - } - else - { - auto concat_arg0 = is_forward ? ret[0] : ret[1]; - auto concat_arg1 = is_forward ? ret[1] : ret[0]; - m.replace_instruction(ins, make_op("concat", {{"axis", 0}}), concat_arg0, concat_arg1); - } - } - - // in case of all sequences are of the same lengths and shorter than the - // max sequence length, need to pad 0's at the end for output hidden states - ins = pad_hidden_states(m, args[0], seq_lens, ins); - replace_last_hs_output(m, ins, seq_lens, last_output, dirct); -} - -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -std::vector rewrite_rnn::gru_cell(bool is_forward, - module& m, - instruction_ref ins, - std::vector inputs, - int linear_before_reset, - const operation& actv_func1, - const operation& actv_func2) const -{ - assert(inputs.size() == 6); - auto seq = inputs.at(0); - auto w = inputs.at(1); - auto r = inputs.at(2); - auto bias = inputs.at(3); - auto seq_lens = inputs.at(4); - auto ih = inputs.at(5); - - instruction_ref hidden_states = m.end(); - instruction_ref last_output{}; - migraphx::shape seq_shape = seq->get_shape(); - migraphx::shape r_shape = r->get_shape(); - long hs = r_shape.lens()[2]; - - migraphx::shape ss(seq_shape.type(), {seq_shape.lens()[1], r_shape.lens()[2]}); - std::vector data(ss.elements(), 1.0f); - auto l1 = m.add_literal(migraphx::literal{ss, data}); - - // w matrix squeeze to 2-dim and do a transpose - std::vector perm{1, 0}; - auto sw = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), w); - auto tw = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), sw); - - // r slide to two part, zr and h - auto sr = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), r); - auto rzr = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {2 * hs}}}), sr); - auto trzr = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), rzr); - - auto rh = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {2 * hs}}, {"ends", {3 * hs}}}), sr); - auto trh = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), rh); - - // initial states - auto sih = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ih); - size_t bs = ih->get_shape().lens()[1]; - - // bias - instruction_ref bwb{}; - instruction_ref brb_zr{}; - instruction_ref brb_h{}; - if(bias != m.end()) - { - auto sbias = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), bias); - auto wb = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {3 * hs}}}), sbias); - bwb = m.insert_instruction( - ins, - make_op("broadcast", {{"axis", 1}, {"out_lens", {bs, static_cast(3 * hs)}}}), - wb); - - auto rb_zr = m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {3 * hs}}, {"ends", {5 * hs}}}), - sbias); - auto rb_h = m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {5 * hs}}, {"ends", {6 * hs}}}), - sbias); - brb_zr = m.insert_instruction( - ins, - make_op("broadcast", {{"axis", 1}, {"out_lens", {bs, static_cast(2 * hs)}}}), - rb_zr); - brb_h = m.insert_instruction( - ins, - make_op("broadcast", {{"axis", 1}, {"out_lens", {bs, static_cast(hs)}}}), - rb_h); - } - - long seq_len = get_seq_len(m, seq, seq_lens); - for(long i = 0; i < seq_len; i++) - { - long seq_index = is_forward ? i : (seq_len - 1 - i); - auto xt = m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {seq_index}}, {"ends", {seq_index + 1}}}), - seq); - auto cont_xt = m.insert_instruction(ins, make_op("contiguous"), xt); - xt = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), cont_xt); - - auto xt_w = m.insert_instruction(ins, make_op("dot"), xt, tw); - auto ih1_rzr = m.insert_instruction(ins, make_op("dot"), sih, trzr); - if(bias != m.end()) - { - xt_w = m.insert_instruction(ins, make_op("add"), xt_w, bwb); - ih1_rzr = m.insert_instruction(ins, make_op("add"), ih1_rzr, brb_zr); - } - - auto xw_z = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {0}}, {"ends", {hs}}}), xt_w); - auto xw_r = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {hs}}, {"ends", {2 * hs}}}), xt_w); - auto xw_h = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {2 * hs}}, {"ends", {3 * hs}}}), xt_w); - - auto hr_z = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {0}}, {"ends", {hs}}}), ih1_rzr); - auto hr_r = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {hs}}, {"ends", {2 * hs}}}), ih1_rzr); - - auto xw_hr_z = m.insert_instruction(ins, make_op("add"), xw_z, hr_z); - auto zt = m.insert_instruction(ins, actv_func1, xw_hr_z); - - auto xw_hr_r = m.insert_instruction(ins, make_op("add"), xw_r, hr_r); - auto rt = m.insert_instruction(ins, actv_func1, xw_hr_r); - - instruction_ref hr_h{}; - if(linear_before_reset == 0) - { - // equation g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) - auto rt_ht1 = m.insert_instruction(ins, make_op("mul"), rt, sih); - hr_h = m.insert_instruction(ins, make_op("dot"), rt_ht1, trh); - if(bias != m.end()) - { - hr_h = m.insert_instruction(ins, make_op("add"), hr_h, brb_h); - } - } - else - { - // equation ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) - auto ht1_rh = m.insert_instruction(ins, make_op("dot"), sih, trh); - if(bias != m.end()) - { - ht1_rh = m.insert_instruction(ins, make_op("add"), ht1_rh, brb_h); - } - hr_h = m.insert_instruction(ins, make_op("mul"), rt, ht1_rh); - } - - auto xw_hr_h = m.insert_instruction(ins, make_op("add"), xw_h, hr_h); - auto ht = m.insert_instruction(ins, actv_func2, xw_hr_h); - - // equation Ht = (1 - zt) (.) ht + zt (.) Ht-1 - auto one_minus_zt = m.insert_instruction(ins, make_op("sub"), l1, zt); - auto one_minus_zt_ht = m.insert_instruction(ins, make_op("mul"), one_minus_zt, ht); - auto zt_ht1 = m.insert_instruction(ins, make_op("mul"), zt, sih); - sih = m.insert_instruction(ins, make_op("add"), one_minus_zt_ht, zt_ht1); - last_output = m.insert_instruction(ins, make_op("unsqueeze", {{"axes", {0, 1}}}), sih); - - if(i < seq_len - 1) - { - if(is_forward) - { - hidden_states = - (seq_index == 0) - ? last_output - : m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), hidden_states, last_output); - } - else - { - hidden_states = - (seq_index == seq_len - 1) - ? last_output - : m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), last_output, hidden_states); - } - } - } - - return {hidden_states, last_output}; -} - -std::vector rewrite_rnn::gru_actv_funcs(instruction_ref ins) const -{ - auto gru_op = any_cast(ins->get_operator()); - // before rewrite the gru operator, need to ensure - // we have 4 actv funcs, even though a user does not - // specifiy any actv func. If less than 4, use the - // algorithm in parse_gru to make 4 actv functions - if(gru_op.direction == op::rnn_direction::bidirectional) - { - if(gru_op.actv_funcs.empty()) - return {make_op("sigmoid"), make_op("tanh"), make_op("sigmoid"), make_op("tanh")}; - else if(gru_op.actv_funcs.size() == 1) - return {gru_op.actv_funcs.at(0), - gru_op.actv_funcs.at(0), - gru_op.actv_funcs.at(0), - gru_op.actv_funcs.at(0)}; - else if(gru_op.actv_funcs.size() == 2) - return {gru_op.actv_funcs.at(0), - gru_op.actv_funcs.at(1), - gru_op.actv_funcs.at(0), - gru_op.actv_funcs.at(1)}; - else if(gru_op.actv_funcs.size() == 3) - return {gru_op.actv_funcs.at(0), - gru_op.actv_funcs.at(1), - gru_op.actv_funcs.at(2), - gru_op.actv_funcs.at(0)}; - else - return gru_op.actv_funcs; - } - else - { - if(gru_op.actv_funcs.empty()) - return {make_op("sigmoid"), make_op("tanh")}; - else if(gru_op.actv_funcs.size() == 1) - return {gru_op.actv_funcs.at(0), gru_op.actv_funcs.at(0)}; - else - return gru_op.actv_funcs; - } -} - -// for lstm operators -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -void rewrite_rnn::apply_lstm(module& m, instruction_ref ins) const -{ - assert(ins->name() == "lstm"); - auto args = ins->inputs(); - - shape seq_shape = args[0]->get_shape(); - std::size_t hidden_size = args[2]->get_shape().lens()[2]; - std::size_t batch_size = seq_shape.lens()[1]; - shape::type_t type = seq_shape.type(); - migraphx::shape ihc_shape{type, {1, batch_size, hidden_size}}; - std::vector ihc_data(ihc_shape.elements(), 0.0); - - migraphx::shape pph_shape{type, {1, 3 * hidden_size}}; - - auto actv_funcs = lstm_actv_funcs(ins); - auto lstm_op = any_cast(ins->get_operator()); - op::rnn_direction dirct = lstm_op.direction; - - // process sequence length - instruction_ref seq_lens = m.end(); - if((args.size() >= 5) and not args[4]->is_undefined()) - { - seq_lens = args[4]; - } - - bool variable_seq_len = is_variable_seq_lens(m, seq_lens); - - instruction_ref last_hs_output{}; - instruction_ref last_cell_output{}; - instruction_ref hidden_state{}; - instruction_ref cell_outputs{}; - if(dirct == op::rnn_direction::bidirectional) - { - // input weight matrix - // input weight matrix - auto w_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[1]); - auto w_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[1]); - - // hidden state weight matrix - auto r_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[2]); - auto r_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[2]); - - // process bias - instruction_ref bias_forward = m.end(); - instruction_ref bias_reverse = m.end(); - if(args.size() >= 4 and not args[3]->is_undefined()) - { - bias_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[3]); - bias_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[3]); - } - - // process intial hidden state, it is the 6th argument - instruction_ref ih_forward{}; - instruction_ref ih_reverse{}; - if(args.size() >= 6 and not args[5]->is_undefined()) - { - ih_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[5]); - ih_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[5]); - } - else - { - ih_forward = m.add_literal(migraphx::literal{ihc_shape, ihc_data}); - ih_reverse = m.add_literal(migraphx::literal{ihc_shape, ihc_data}); - } - - // process initial cell value - instruction_ref ic_forward{}; - instruction_ref ic_reverse{}; - if(args.size() >= 7 and not args[6]->is_undefined()) - { - ic_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[6]); - ic_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[6]); - } - else - { - ic_forward = m.add_literal(migraphx::literal{ihc_shape, ihc_data}); - ic_reverse = m.add_literal(migraphx::literal{ihc_shape, ihc_data}); - } - - // process weight of the peephole - instruction_ref pph_forward = m.end(); - instruction_ref pph_reverse = m.end(); - if(args.size() == 8 and not args[7]->is_undefined()) - { - pph_forward = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), args[7]); - pph_reverse = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), args[7]); - } - - auto ret_forward = lstm_cell(true, - m, - ins, - {args[0], - w_forward, - r_forward, - bias_forward, - seq_lens, - ih_forward, - ic_forward, - pph_forward}, - actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(2)); - - if(variable_seq_len) - { - args[0] = - m.insert_instruction(ins, make_op("rnn_var_sl_shift_sequence"), args[0], seq_lens); - } - auto ret_reverse = lstm_cell(false, - m, - ins, - {args[0], - w_reverse, - r_reverse, - bias_reverse, - seq_lens, - ih_reverse, - ic_reverse, - pph_reverse}, - actv_funcs.at(3), - actv_funcs.at(4), - actv_funcs.at(5)); - - auto concat_hs_output = m.insert_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[1], ret_reverse[1]); - auto concat_cell_output = m.insert_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[3], ret_reverse[3]); - last_hs_output = - m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), concat_hs_output); - last_cell_output = - m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), concat_cell_output); - - // the following logic is to ensure the last instruction is a concat - if(ret_forward[0] == m.end()) - { - cell_outputs = concat_cell_output; - } - else - { - ret_forward[1] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_forward[0], ret_forward[1]); - ret_reverse[1] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_reverse[1], ret_reverse[0]); - - ret_forward[3] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_forward[2], ret_forward[3]); - ret_reverse[3] = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), ret_reverse[3], ret_reverse[2]); - cell_outputs = m.insert_instruction( - ins, make_op("concat", {{"axis", 1}}), ret_forward[3], ret_reverse[3]); - } - - hidden_state = m.replace_instruction( - ins, make_op("concat", {{"axis", 1}}), {ret_forward[1], ret_reverse[1]}); - } - else - { - bool is_forward = (dirct == op::rnn_direction::forward); - // weight matrices - auto w = args[1]; - auto r = args[2]; - - // bias - instruction_ref bias = m.end(); - if(args.size() >= 4 and not args[3]->is_undefined()) - { - bias = args[3]; - } - - // initial hidden state - instruction_ref ih{}; - if(args.size() >= 6 and not args[5]->is_undefined()) - { - ih = args[5]; - } - else - { - ih = m.add_literal(migraphx::literal{ihc_shape, ihc_data}); - } - - // initial cell value - instruction_ref ic{}; - if(args.size() >= 7 and not args[6]->is_undefined()) - { - ic = args[6]; - } - else - { - ic = m.add_literal(migraphx::literal{ihc_shape, ihc_data}); - } - - // process weight of the peephole - instruction_ref pph = m.end(); - if(args.size() == 8 and not args[7]->is_undefined()) - { - pph = args[7]; - } - - if(not is_forward and variable_seq_len) - { - args[0] = - m.insert_instruction(ins, make_op("rnn_var_sl_shift_sequence"), args[0], seq_lens); - } - auto ret = lstm_cell(is_forward, - m, - ins, - {args[0], w, r, bias, seq_lens, ih, ic, pph}, - actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(2)); - - last_hs_output = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ret[1]); - last_cell_output = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ret[3]); - - if(ret[0] == m.end()) - { - cell_outputs = ret[3]; - hidden_state = m.replace_instruction(ins, make_op("concat", {{"axis", 0}}), ret[1]); - } - else - { - auto concat_cell_arg0 = is_forward ? ret[2] : ret[3]; - auto concat_cell_arg1 = is_forward ? ret[3] : ret[2]; - cell_outputs = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), concat_cell_arg0, concat_cell_arg1); - - auto concat_arg0 = is_forward ? ret[0] : ret[1]; - auto concat_arg1 = is_forward ? ret[1] : ret[0]; - hidden_state = m.replace_instruction( - ins, make_op("concat", {{"axis", 0}}), concat_arg0, concat_arg1); - } - } - - // in case of all sequences are of the same lengths and shorter than the - // max sequence length, need to pad 0's at the end for output hidden states - hidden_state = pad_hidden_states(m, args[0], seq_lens, hidden_state); - - // replace last hidden states with corresponding instructions - ins = replace_last_hs_output(m, hidden_state, seq_lens, last_hs_output, dirct); - - // replace last cell outputs with corresponding instructions - replace_last_cell_output(m, ins, seq_lens, cell_outputs, last_cell_output, dirct); -} - -// NOLINTNEXTLINE(readability-function-cognitive-complexity) -std::vector rewrite_rnn::lstm_cell(bool is_forward, - module& m, - instruction_ref ins, - std::vector inputs, - const operation& actv_func1, - const operation& actv_func2, - const operation& actv_func3) const -{ - // must have 7 args in the input vector - assert(inputs.size() == 8); - auto seq = inputs.at(0); - auto w = inputs.at(1); - auto r = inputs.at(2); - auto bias = inputs.at(3); - auto seq_lens = inputs.at(4); - auto ih = inputs.at(5); - auto ic = inputs.at(6); - auto pph = inputs.at(7); - - instruction_ref hidden_states = m.end(); - instruction_ref cell_outputs = m.end(); - - instruction_ref last_hs_output{}; - instruction_ref last_cell_output{}; - - migraphx::shape r_shape = r->get_shape(); - long hs = r_shape.lens()[2]; - auto bs = ih->get_shape().lens()[1]; - - std::vector perm{1, 0}; - // w matrix, squeeze and transpose - auto sw = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), w); - auto tsw = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), sw); - - // r matrix, squeeze and transpose - auto sr = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), r); - auto tsr = m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), sr); - - // initial hidden state - auto sih = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ih); - - // initial cell state - auto sic = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), ic); - auto ic_lens = sic->get_shape().lens(); - - // bias - instruction_ref wrb{}; - if(bias != m.end()) - { - - auto sbias = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), bias); - auto ub_wb = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {4 * hs}}}), sbias); - auto ub_rb = m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {4 * hs}}, {"ends", {8 * hs}}}), - sbias); - auto ub_wrb = m.insert_instruction(ins, make_op("add"), ub_wb, ub_rb); - - wrb = m.insert_instruction( - ins, - make_op("broadcast", {{"axis", 1}, {"out_lens", {bs, 4 * static_cast(hs)}}}), - ub_wrb); - } - - // peep hole - instruction_ref pphi_brcst{}; - instruction_ref ppho_brcst{}; - instruction_ref pphf_brcst{}; - if(pph != m.end()) - { - auto spph = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), pph); - auto pphi = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {hs}}}), spph); - pphi_brcst = m.insert_instruction( - ins, make_op("broadcast", {{"axis", 1}, {"out_lens", ic_lens}}), pphi); - - auto ppho = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {hs}}, {"ends", {2 * hs}}}), spph); - ppho_brcst = m.insert_instruction( - ins, make_op("broadcast", {{"axis", 1}, {"out_lens", ic_lens}}), ppho); - - auto pphf = m.insert_instruction( - ins, make_op("slice", {{"axes", {0}}, {"starts", {2 * hs}}, {"ends", {3 * hs}}}), spph); - pphf_brcst = m.insert_instruction( - ins, make_op("broadcast", {{"axis", 1}, {"out_lens", ic_lens}}), pphf); - } - - long seq_len = get_seq_len(m, seq, seq_lens); - for(long i = 0; i < seq_len; ++i) - { - long seq_index = is_forward ? i : (seq_len - 1 - i); - auto xt = m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {seq_index}}, {"ends", {seq_index + 1}}}), - seq); - auto cont_xt = m.insert_instruction(ins, make_op("contiguous"), xt); - xt = m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), cont_xt); - - auto xt_tsw = m.insert_instruction(ins, make_op("dot"), xt, tsw); - auto sih_tsr = m.insert_instruction(ins, make_op("dot"), sih, tsr); - auto xt_sih = m.insert_instruction(ins, make_op("add"), xt_tsw, sih_tsr); - if(bias != m.end()) - { - xt_sih = m.insert_instruction(ins, make_op("add"), xt_sih, wrb); - } - - auto it_before_actv = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {0}}, {"ends", {hs}}}), xt_sih); - auto ot_before_actv = m.insert_instruction( - ins, make_op("slice", {{"axes", {1}}, {"starts", {hs}}, {"ends", {2 * hs}}}), xt_sih); - auto ft_before_actv = m.insert_instruction( - ins, - make_op("slice", {{"axes", {1}}, {"starts", {2 * hs}}, {"ends", {3 * hs}}}), - xt_sih); - auto ct_before_actv = m.insert_instruction( - ins, - make_op("slice", {{"axes", {1}}, {"starts", {3 * hs}}, {"ends", {4 * hs}}}), - xt_sih); - - if(pph != m.end()) - { - auto pphi_ct = m.insert_instruction(ins, make_op("mul"), pphi_brcst, sic); - it_before_actv = m.insert_instruction(ins, make_op("add"), it_before_actv, pphi_ct); - - auto pphf_ct = m.insert_instruction(ins, make_op("mul"), pphf_brcst, sic); - ft_before_actv = m.insert_instruction(ins, make_op("add"), ft_before_actv, pphf_ct); - } - auto it = m.insert_instruction(ins, actv_func1, it_before_actv); - auto ft = m.insert_instruction(ins, actv_func1, ft_before_actv); - auto ct = m.insert_instruction(ins, actv_func2, ct_before_actv); - - // equation Ct = ft (.) Ct-1 + it (.) ct - auto ft_cell = m.insert_instruction(ins, make_op("mul"), ft, sic); - auto it_ct = m.insert_instruction(ins, make_op("mul"), it, ct); - auto cellt = m.insert_instruction(ins, make_op("add"), ft_cell, it_ct); - - if(pph != m.end()) - { - auto ppho_cellt = m.insert_instruction(ins, make_op("mul"), ppho_brcst, cellt); - ot_before_actv = m.insert_instruction(ins, make_op("add"), ot_before_actv, ppho_cellt); - } - auto ot = m.insert_instruction(ins, actv_func1, ot_before_actv); - - // Ht = ot (.) h(Ct) - auto h_cellt = m.insert_instruction(ins, actv_func3, cellt); - auto ht = m.insert_instruction(ins, make_op("mul"), ot, h_cellt); - - sic = cellt; - sih = ht; - - last_hs_output = m.insert_instruction(ins, make_op("unsqueeze", {{"axes", {0, 1}}}), ht); - last_cell_output = - m.insert_instruction(ins, make_op("unsqueeze", {{"axes", {0, 1}}}), cellt); - - if(i < seq_len - 1) - { - if(i == 0) - { - hidden_states = last_hs_output; - cell_outputs = last_cell_output; - } - else - { - auto concat_hs_arg0 = is_forward ? hidden_states : last_hs_output; - auto concat_hs_arg1 = is_forward ? last_hs_output : hidden_states; - hidden_states = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), concat_hs_arg0, concat_hs_arg1); - - auto concat_cell_arg0 = is_forward ? cell_outputs : last_cell_output; - auto concat_cell_arg1 = is_forward ? last_cell_output : cell_outputs; - cell_outputs = m.insert_instruction( - ins, make_op("concat", {{"axis", 0}}), concat_cell_arg0, concat_cell_arg1); - } - } - } - - return {hidden_states, last_hs_output, cell_outputs, last_cell_output}; -} - -std::vector rewrite_rnn::lstm_actv_funcs(instruction_ref ins) const -{ - auto lstm_op = any_cast(ins->get_operator()); - // before rewrite the lstm operator, need to ensure - // we have 6 actv funcs, even though a user does not - // specifiy any actv func. If less than 46, use the - // algorithm in parse_lstm to make 6 actv functions - const auto& actv_funcs = lstm_op.actv_funcs; - std::size_t num_actv_funcs = actv_funcs.size(); - if(lstm_op.direction == op::rnn_direction::bidirectional) - { - switch(num_actv_funcs) - { - case 0: - return {make_op("sigmoid"), - make_op("tanh"), - make_op("tanh"), - make_op("sigmoid"), - make_op("tanh"), - make_op("tanh")}; - - case 1: - return {actv_funcs.at(0), - actv_funcs.at(0), - actv_funcs.at(0), - actv_funcs.at(0), - actv_funcs.at(0), - actv_funcs.at(0)}; - - case 2: - return {actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(1), - actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(1)}; - - case 3: - return {actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(2), - actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(2)}; - - case 4: - return {actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(2), - actv_funcs.at(3), - actv_funcs.at(3), - actv_funcs.at(3)}; - - case 5: - return {actv_funcs.at(0), - actv_funcs.at(1), - actv_funcs.at(2), - actv_funcs.at(3), - actv_funcs.at(4), - actv_funcs.at(4)}; - - default: return actv_funcs; - } - } - else - { - switch(num_actv_funcs) - { - case 0: return {make_op("sigmoid"), make_op("tanh"), make_op("tanh")}; - - case 1: return {actv_funcs.at(0), actv_funcs.at(0), actv_funcs.at(0)}; - - case 2: return {actv_funcs.at(0), actv_funcs.at(1), actv_funcs.at(1)}; - - default: return actv_funcs; - } - } -} - -bool rewrite_rnn::is_variable_seq_lens(const module& m, instruction_ref seq_lens) const -{ - bool is_var_lens = false; - if(seq_lens != m.end()) - { - if(seq_lens->can_eval()) - { - auto arg_lens = seq_lens->eval(); - std::vector vec_lens; - arg_lens.visit([&](auto l) { vec_lens.assign(l.begin(), l.end()); }); - int64_t l = 0; - if(not vec_lens.empty()) - { - l = vec_lens[0]; - } - if(not std::all_of(vec_lens.begin(), vec_lens.end(), [&](auto v) { return v == l; })) - { - is_var_lens = true; - } - } - else - { - is_var_lens = true; - } - } - - return is_var_lens; -} - -std::size_t -rewrite_rnn::get_seq_len(const module& m, instruction_ref input, instruction_ref seq_lens) const -{ - bool is_var_lens = is_variable_seq_lens(m, seq_lens); - auto input_shape = input->get_shape(); - auto length = input_shape.lens()[0]; - if(not is_var_lens and seq_lens != m.end()) - { - auto arg_len = seq_lens->eval(); - std::vector vec_lens; - arg_len.visit([&](auto l) { vec_lens.assign(l.begin(), l.end()); }); - length = vec_lens.empty() ? length : vec_lens[0]; - } - - return length; -} - -instruction_ref rewrite_rnn::replace_last_hs_output(module& m, - instruction_ref ins, - instruction_ref seq_lens, - instruction_ref last_hs_output, - op::rnn_direction dirct) const -{ - bool variable_seq_len = is_variable_seq_lens(m, seq_lens); - instruction_ref result_ins{}; - if(variable_seq_len) - { - result_ins = - m.insert_instruction(std::next(ins), - make_op("rnn_var_sl_shift_output", - {{"output_name", "hidden_states"}, {"direction", dirct}}), - ins, - seq_lens); - m.replace_instruction(ins, result_ins); - auto hs_outputs = find_all(result_ins->outputs(), - [&](auto i) { return i->name() == "rnn_last_hs_output"; }); - - for(auto& hs_out : hs_outputs) - { - auto inputs = hs_out->inputs(); - m.replace_instruction(hs_out, - make_op("rnn_var_sl_last_output", {{"direction", dirct}}), - inputs.front(), - seq_lens); - } - } - else - { - auto hs_outputs = - find_all(ins->outputs(), [&](auto i) { return i->name() == "rnn_last_hs_output"; }); - - for(auto& hs_out : hs_outputs) - { - m.replace_instruction(hs_out, last_hs_output); - } - - result_ins = ins; - } - - return result_ins; -} - -void rewrite_rnn::replace_last_cell_output(module& m, - instruction_ref ins, - instruction_ref seq_lens, - instruction_ref cell_outputs, - instruction_ref last_cell_output, - op::rnn_direction dirct) const -{ - bool variable_seq_len = is_variable_seq_lens(m, seq_lens); - auto ins_outputs = - find_all(ins->outputs(), [&](auto i) { return i->name() == "rnn_last_cell_output"; }); - - if(variable_seq_len) - { - if(not ins_outputs.empty()) - { - cell_outputs = m.insert_instruction( - std::next(ins), - make_op("rnn_var_sl_shift_output", - {{"output_name", "cell_outputs"}, {"direction", dirct}}), - cell_outputs, - seq_lens); - } - - for(auto co : ins_outputs) - { - m.replace_instruction(co, - make_op("rnn_var_sl_last_output", {{"direction", dirct}}), - cell_outputs, - seq_lens); - } - } - // replace the rnn_last_cell_output with the last_cell_output. The while - // loop is to handle the case of multiple rnn_last_cell_output operators - else - { - for(auto co : ins_outputs) - { - m.replace_instruction(co, last_cell_output); - } - } -} - -instruction_ref rewrite_rnn::pad_hidden_states(module& m, - instruction_ref seq, - instruction_ref seq_lens, - instruction_ref hs) const -{ - auto max_seq_len = seq->get_shape().lens()[0]; - auto seq_len = get_seq_len(m, seq, seq_lens); - - // condition of all sequence are of the same length and - // less than max_seq_len, we need to append the hs outputs - auto hs_padded = hs; - if(seq_len < max_seq_len) - { - auto s = hs->get_shape(); - auto pad_lens = s.lens(); - pad_lens[0] = static_cast(max_seq_len - seq_len); - shape pad_s{s.type(), pad_lens}; - std::vector pad_data(pad_s.elements(), 0.0f); - auto pl = m.add_literal(pad_s, pad_data.begin(), pad_data.end()); - hs_padded = m.insert_instruction(std::next(hs), make_op("concat", {{"axis", 0}}), hs, pl); - m.replace_instruction(hs, hs_padded); - } - - return hs_padded; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/schedule.cpp b/docker/rocm/migraphx/schedule.cpp deleted file mode 100644 index 0b3670a05..000000000 --- a/docker/rocm/migraphx/schedule.cpp +++ /dev/null @@ -1,633 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_SCHEDULE) - -auto get_inputs() -{ - return [](auto i) { return i->inputs(); }; -} - -auto get_outputs() -{ - return [](auto i) { return i->outputs(); }; -} - -struct stream_info -{ - std::unordered_map ins2stream; - std::unordered_map weights; - std::unordered_map iweights; - ins_dep_map mod_implicit_deps; - - void calc_implicit_deps(const module& m) { mod_implicit_deps = m.calc_implicit_deps(); } - - void accumulate_weights(instruction_ref last, const schedule_model& model) - { - fix([&](auto self, auto ins) -> std::size_t { - if(not contains(weights, ins)) - { - std::size_t weight = 0; - auto&& op = ins->get_operator(); - if(not is_context_free(op) and op.name()[0] != '@') - weight = model.weight(op); - // This will ensure a stream will be assigned to return - if(op.name() == "@return") - weight = 1; - iweights[ins] = weight; - auto inputs = ins->inputs(); - if(contains(mod_implicit_deps, ins)) - { - const auto& impl_deps = mod_implicit_deps.at(ins); - inputs.insert(inputs.end(), impl_deps.begin(), impl_deps.end()); - } - - weights[ins] = std::accumulate( - inputs.begin(), inputs.end(), weight, [&](std::size_t w, instruction_ref i) { - return w + self(i); - }); - } - return weights[ins]; - })(last); - } - - template - void sort_args_by_weight(std::vector& args, Compare compare) const - { - if(args.size() < 2) - return; - std::sort(args.begin(), args.end(), by(compare, [this](auto x) { - return std::make_tuple( - this->weights.at(x), x->inputs().size(), std::addressof(*x)); - })); - } - - std::vector::iterator sort_args(std::vector& args) - { - if(args.size() < 2) - { - return args.end(); - } - - const std::size_t min_partition_threshold = 2; - sort_args_by_weight(args, std::greater<>{}); - - auto it = std::lower_bound(std::next(args.begin()), - args.end(), - min_partition_threshold, - [&](auto i, std::size_t w) { return this->weights[i] > w; }); - assert(it == args.end() or this->weights[*it] <= min_partition_threshold); - assert(it == args.end() or std::prev(it) == args.begin() or - this->weights[*std::prev(it)] > min_partition_threshold); - return it; - } - - struct partition - { - std::size_t weight = 0; - std::vector instructions{}; - - void add(instruction_ref ins, std::size_t w) - { - weight += w; - instructions.push_back(ins); - } - }; - - std::size_t assign_streams(module& m, std::size_t n) - { - assert(n > 0); - partition critical; - std::unordered_map> partitions; - partitions.reserve(weights.size()); - fix([&](auto self, auto ins, auto& part) { - assert(not is_end(ins, m.end())); - if(not m.has_instruction(ins)) - return; - if(contains(partitions, ins)) - return; - - // Add an entry so we know the instruction was visited - partitions[ins]; - part.add(ins, this->iweights[ins]); - - auto args = ins->inputs(); - auto threshold_it = this->sort_args(args); - - if(not args.empty()) - { - assert(threshold_it != args.begin()); - self(args.front(), part); - for(auto i : range(std::next(args.begin()), threshold_it)) - { - partitions[ins].emplace_back(); - self(i, partitions[ins].back()); - } - for(auto i : range(threshold_it, args.end())) - { - self(i, part); - } - } - // Sort instructions - m.move_instruction(ins, m.end()); - })(std::prev(m.end()), critical); - - // Set the critical partition to stream 0 - set_stream(critical, 0); - if(n == 1) - { - // Assign streams for the other partitions - for(auto&& ins_part : partitions) - for(auto&& part : ins_part.second) - set_stream(part, 0); - return 1; - } - else - { - std::vector streams(n - 1); - // Assign streams for the other partitions - for(auto&& ins_part : partitions) - { - std::sort(ins_part.second.begin(), - ins_part.second.end(), - by(std::greater<>{}, [](auto&& x) { - return std::make_tuple(x.weight, x.instructions.size()); - })); - for(auto&& part : ins_part.second) - { - auto stream = - std::min_element(streams.begin(), streams.end()) - streams.begin(); - set_stream(part, stream + 1); - streams[stream] += part.weight; - } - } - return 1 + std::count_if(streams.begin(), streams.end(), [](auto x) { return x > 0; }); - } - } - - using weight_ins = std::pair; - struct compare_weight_ins - { - bool operator()(const weight_ins& x, const weight_ins& y) const - { - return std::make_pair(x.first, std::addressof(*x.second)) < - std::make_pair(y.first, std::addressof(*y.second)); - } - }; - - void sort(module& m, std::size_t) - { - std::set children; - std::unordered_map visited; - auto last = std::prev(m.end()); - auto mw = this->weights.at(last); - auto nw = mw / (m.size() + 1); - auto add_child = [&](auto ins) { - auto x = 1 + (mw - this->weights.at(ins)) / (nw + 1); - auto w = x * this->iweights.at(ins); - auto& v = visited[ins]; - auto it = children.find(std::make_pair(v * w, ins)); - if(it == children.end()) - { - v++; - children.insert(std::make_pair(v * w, ins)); - } - }; - add_child(last); - - while(not children.empty()) - { - // Pop the first element - auto top = children.begin()->second; - children.erase(children.begin()); - m.move_instruction(top, m.begin()); - for(auto ins : top->inputs()) - { - if(not m.has_instruction(ins)) - continue; - add_child(ins); - } - - if(contains(mod_implicit_deps, top)) - { - for(auto ins : mod_implicit_deps.at(top)) - { - assert(m.has_instruction(ins)); - add_child(ins); - } - } - } - - // move dangling parameter to the front so as not be removed - auto ins = std::next(last); - while(ins != m.end()) - { - auto next = std::next(ins); - if(ins->name() == "@param") - { - m.move_instruction(ins, m.begin()); - } - ins = next; - } - } - - void set_stream(const partition& p, std::size_t n) - { - for(auto ins : p.instructions) - if(iweights[ins] > 0) - set_stream(ins, n); - } - - void set_stream(instruction_ref ins, std::size_t n) - { - assert(iweights[ins] > 0); - ins2stream[ins] = n; - } - - std::size_t get_stream(instruction_ref ins) const { return ins2stream.at(ins); } - - bool has_stream(instruction_ref ins) const { return contains(ins2stream, ins); } - - template - bool different(F f, std::size_t stream) const - { - bool result = false; - f([&](auto s) { - if(s != stream) - { - result = true; - return false; - } - // cppcheck-suppress uselessAssignmentArg - stream = s; - return true; - }); - return result; - } - - template - bool different(F f) const - { - bool result = false; - f([&](auto s) { - result = this->different(f, s); - return false; - }); - return result; - } - - template - auto get_streams_from(instruction_ref start, Selector select) const - { - return [=](auto f) { - return fix([&](auto self, auto ins) { - return all_of(select(ins), [&](auto i) { - if(has_stream(i)) - return f(this->get_stream(i)); - else - return self(i); - }); - })(start); - }; - } - - std::unordered_set get_streams(instruction_ref ins) const - { - if(has_stream(ins)) - return {get_stream(ins)}; - std::unordered_set result; - get_streams_from(ins, get_inputs())([&](auto s) { - result.insert(s); - return true; - }); - return result; - } - - template - bool is_merge_point(instruction_ref ins, Ts... xs) const - { - return different(get_streams_from(ins, get_inputs()), xs...); - } - - template - bool is_split_point(instruction_ref ins, Ts... xs) const - { - return different(get_streams_from(ins, get_outputs()), xs...); - } - - std::vector get_recorded_instructions(instruction_ref start) - { - std::vector result; - std::unordered_map m; - fix([&](auto self, auto ins) { - for(auto i : ins->inputs()) - { - if(iweights.at(i) == 0) - { - self(i); - continue; - } - auto stream = this->get_stream(i); - if(not contains(m, stream)) - m[stream] = i; - else - m[stream] = std::min(m[stream], i, by(std::less<>{}, [&](auto x) { - return std::distance(x, start); - })); - } - })(start); - std::transform( - m.begin(), m.end(), std::back_inserter(result), [](auto&& p) { return p.second; }); - return result; - } - - std::unordered_map>> - find_concurrent_instructions(module& m) const - { - std::unordered_map>> result; - std::unordered_map> merge_from; - dominator_info di = compute_dominator(m); - result.reserve(m.size()); - merge_from.reserve(m.size()); - for(auto ins : reverse_iterator_for(m)) - { - for(auto&& arg : ins->outputs()) - { - if(not m.has_instruction(arg)) - continue; - if(is_merge_point(arg)) - merge_from[ins].insert(arg); - merge_from[ins].insert(merge_from[arg].begin(), merge_from[arg].end()); - } - - if(is_split_point(ins)) - { - erase_if(merge_from[ins], - [&](auto merge) { return di.strictly_dominate(ins, merge); }); - } - - auto streams = this->get_streams(ins); - // Collect concur instructions for each merge point. - for(const auto& merge : merge_from[ins]) - { - for(auto stream : streams) - { - if(result[merge].size() <= stream) - result[merge].resize(stream + 1); - auto&& r = result[merge][stream]; - r.push_back(ins); - // Copy inputs if they dont have a stream(and are not a builtin and context - // free). Inputs without a stream can have a implicit dependency - std::copy_if(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(r), - [&](auto x) { - return not this->has_stream(x) and - not is_context_free(x->get_operator()) and - x->name().front() != '@'; - }); - } - } - } - return result; - } - - std::unordered_map> - get_conflicts(module& m) - { - - using conflict_table_type = - std::unordered_map>; - conflict_table_type conflict_table; - auto concur_ins = this->find_concurrent_instructions(m); - - // Compute an index for each instruction - std::unordered_map ins2index; - std::size_t index_total = 0; - for(auto ins : iterator_for(m)) - ins2index[ins] = index_total++; - - std::vector thread_conflict_tables( - std::thread::hardware_concurrency()); - std::vector index_to_ins; - index_to_ins.reserve(concur_ins.size()); - std::transform(concur_ins.begin(), - concur_ins.end(), - std::back_inserter(index_to_ins), - [](auto&& it) { return it.first; }); - - simple_par_for(concur_ins.size(), [&](auto ins_index, auto tid) { - auto merge_first = index_to_ins[ins_index]; - assert(concur_ins.count(merge_first) > 0); - auto& merge_second = concur_ins.at(merge_first); - - // ensure there are enough elements for different threads - assert(tid < thread_conflict_tables.size()); - auto& thrd_table = thread_conflict_tables.at(tid); - - std::unordered_set checked_ins_set; - auto range_i = range(merge_second.begin(), std::prev(merge_second.end())); - for(auto it_i : iterator_for(range_i)) - { - std::unordered_set ins1_set; - std::copy_if(it_i->begin(), - it_i->end(), - std::inserter(ins1_set, ins1_set.end()), - [&](auto i) { return not contains(checked_ins_set, i); }); - checked_ins_set.insert(ins1_set.begin(), ins1_set.end()); - - auto range_j = range(std::next(it_i), merge_second.end()); - std::unordered_set ins2_set; - for(auto it_j : iterator_for(range_j)) - { - std::copy_if(it_j->begin(), - it_j->end(), - std::inserter(ins2_set, ins2_set.end()), - [&](auto i) { return not contains(checked_ins_set, i); }); - } - - for(auto ins1 : ins1_set) - { - auto p1 = ins2index.at(ins1); - for(auto ins2 : ins2_set) - { - if(ins1 == ins2) - continue; - auto p2 = ins2index.at(ins2); - if(p2 > p1) - thrd_table[ins2].insert(ins1); - else - thrd_table[ins1].insert(ins2); - } - } - } - }); - - // merge thread_conflict_tables together - for(auto& tbl : thread_conflict_tables) - { - for(auto& it : tbl) - { - conflict_table[it.first].insert(it.second.begin(), it.second.end()); - } - } - - // Remove instructions from the conflict table of an ealier instruction - for(auto&& ip : conflict_table) - { - auto ins1 = ip.first; - for(auto ins2 : ip.second) - if(contains(conflict_table[ins2], ins1)) - conflict_table[ins2].erase(ins1); - } - - return conflict_table; - } -}; - -void schedule::apply(module& m) const -{ - if(not enable) - return; - - stream_info si; - si.calc_implicit_deps(m); - auto last = std::prev(m.end()); - si.accumulate_weights(last, model); - auto nstreams = si.assign_streams(m, model.concurrency()); - si.sort(m, model.concurrency()); - - if(enabled(MIGRAPHX_TRACE_COMPILE{}) or enabled(MIGRAPHX_TRACE_SCHEDULE{})) - { - m.annotate(std::cout, [&](auto ins) { - if(ins->name() == "@param" and not contains(si.weights, ins)) - return; - - std::cout << ":"; - std::cout << " weight=" << si.weights.at(ins); - std::cout << " input={"; - si.get_streams_from(ins, get_inputs())([&](auto s) { - std::cout << s << ","; - return true; - }); - std::cout << "}"; - if(si.has_stream(ins)) - std::cout << " stream=" << si.get_stream(ins); - }); - std::cout << std::endl; - } - - // No concurrency - if(nstreams < 2) - return; - - // Schedule instructions - std::size_t wait_id = 0; - std::unordered_map ins2wait; - std::unordered_map> waited_for; - std::unordered_map> ins2waited; - ins2wait.reserve(m.size()); - ins2waited.reserve(m.size()); - for(auto ins : iterator_for(m)) - { - // Only schedule instructions that have a stream - if(not si.has_stream(ins)) - continue; - assert(si.weights[ins] > 0); - // Schedule instruction on the stream - auto stream = si.get_stream(ins); - assert(stream < model.concurrency()); - model.sched(m, ins, stream); - // Insert wait instructions - if(si.is_merge_point(ins, stream)) - { - for(auto i : si.get_recorded_instructions(ins)) - { - if(not si.has_stream(i) or si.get_stream(i) == stream) - continue; - - // Create a new event if it hasn't been recorded - if(not contains(ins2wait, i)) - { - ins2wait[i] = wait_id; - model.record(m, i, wait_id); - wait_id++; - } - auto w = ins2wait.at(i); - // If we already waited for the event on this stream then dont - // insert another wait event - if(not contains(waited_for[stream], w)) - model.wait(m, ins, w); - // Store the event as waited - waited_for[stream].insert(w); - // Store all wait events that have been waited on prior to the recorded instruction - waited_for[stream].insert(ins2waited[i].begin(), ins2waited[i].end()); - } - } - // Store wait events that have already been waited on - if(si.is_split_point(ins, stream)) - { - ins2waited[ins] = waited_for[stream]; - } - } - - // Add memory conflicts - auto conflict_table = si.get_conflicts(m); - for(auto&& ip : conflict_table) - { - if(ip.second.empty()) - continue; - std::vector args; - args.push_back(ip.first); - args.insert(args.end(), ip.second.begin(), ip.second.end()); - m.insert_instruction(std::next(ip.first), make_op("identity"), args); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/serialize.cpp b/docker/rocm/migraphx/serialize.cpp deleted file mode 100644 index eff944be0..000000000 --- a/docker/rocm/migraphx/serialize.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -void raw_data_to_value(value& v, const RawData& rd) -{ - value result; - result["shape"] = migraphx::to_value(rd.get_shape()); - if(rd.get_shape().type() == shape::tuple_type) - result["sub"] = migraphx::to_value(rd.get_sub_objects()); - else if(not rd.empty()) - result["data"] = migraphx::value::binary(rd.data(), rd.get_shape().bytes()); - v = result; -} - -void migraphx_to_value(value& v, const literal& l) { raw_data_to_value(v, l); } -void migraphx_from_value(const value& v, literal& l) -{ - auto s = migraphx::from_value(v.at("shape")); - l = literal(s, v.at("data").get_binary().data()); -} - -void migraphx_to_value(value& v, const argument& a) { raw_data_to_value(v, a); } -void migraphx_from_value(const value& v, argument& a) -{ - if(v.contains("data")) - { - literal l = migraphx::from_value(v); - a = l.get_argument(); - } - else if(v.contains("sub")) - { - a = migraphx::from_value>(v.at("sub")); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/shape.cpp b/docker/rocm/migraphx/shape.cpp deleted file mode 100644 index fd8173f0b..000000000 --- a/docker/rocm/migraphx/shape.cpp +++ /dev/null @@ -1,857 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct shape_impl -{ - static std::shared_ptr default_shape() - { - static const std::shared_ptr result = std::make_shared(); - return result; - } - - shape_impl() : m_type(shape::float_type) {} - - shape_impl(shape::type_t t) : m_type(t), m_lens({1}), m_strides({0}), m_standard(true) - { - assert(t != shape::tuple_type); - } - - shape_impl(shape::type_t t, std::vector l) - : m_type(t), m_lens(std::move(l)), m_standard(true) - { - assert(t != shape::tuple_type); - this->calculate_strides(); - } - - shape_impl(shape::type_t t, std::vector l, std::vector s) - : m_type(t), m_lens(std::move(l)), m_strides(std::move(s)) - { - assert(t != shape::tuple_type); - assert(m_lens.size() == m_strides.size()); - - // Calculate standard shape flag for these lens/strides. Strides on size-1 - // axes are ignored to support an MLIR rule. - std::vector filtered_strides; - for(size_t ind = 0; ind < m_strides.size(); ind++) - if(m_lens[ind] != 1) - filtered_strides.push_back(m_strides[ind]); - m_standard = this->elements() == this->element_space() and not skips() and - std::is_sorted(filtered_strides.rbegin(), filtered_strides.rend()); - } - - shape_impl(shape::type_t t, std::vector dims) - : m_type(t), m_dyn_dims(std::move(dims)) - { - } - - shape_impl(shape::type_t t, - std::vector mins, - std::vector maxes, - std::vector> optimals_list) - : m_type(t) - { - if(optimals_list.empty()) - { - for(size_t i = 0; i < mins.size(); ++i) - { - m_dyn_dims.push_back(shape::dynamic_dimension{mins[i], maxes[i]}); - } - } - else - { - assert(mins.size() == maxes.size() and maxes.size() == optimals_list.size()); - for(size_t i = 0; i < mins.size(); ++i) - { - m_dyn_dims.push_back(shape::dynamic_dimension{mins[i], maxes[i], optimals_list[i]}); - } - } - } - - shape_impl(const std::vector& subs) : m_type(shape::tuple_type), m_shapes(subs) {} - - shape::type_t m_type; - std::vector m_lens = {}; - std::vector m_strides = {}; - std::vector m_shapes = {}; - bool m_standard = false; - - std::vector m_dyn_dims = {}; - - void calculate_strides() - { - m_strides.clear(); - m_strides.resize(m_lens.size(), 0); - if(m_strides.empty()) - return; - m_strides.back() = 1; - std::partial_sum(m_lens.rbegin(), - m_lens.rend() - 1, - m_strides.rbegin() + 1, - std::multiplies()); - } - - std::size_t element_space() const - { - if(not m_dyn_dims.empty()) - { - auto maxes = max_lens(); - std::size_t max_val = std::numeric_limits::max(); - - return std::accumulate( - maxes.begin(), maxes.end(), std::size_t{1}, [&](std::size_t x, std::size_t y) { - // overflow check and clip - if(x != 0 and y > max_val / x) - { - return max_val; - } - return x * y; - }); - } - - assert(m_lens.size() == m_strides.size()); - if(m_lens.empty()) - return 0; - return std::inner_product(m_lens.begin(), - m_lens.end(), - m_strides.begin(), - std::size_t{0}, - std::plus{}, - [](std::size_t l, std::size_t s) { return (l - 1) * s; }) + - 1; - } - - std::size_t elements() const - { - if(not m_dyn_dims.empty()) - { - MIGRAPHX_THROW("SHAPE: elements() called on dynamic shape"); - } - - assert(m_lens.size() == m_strides.size()); - if(m_lens.empty()) - return 0; - return std::accumulate( - m_lens.begin(), m_lens.end(), std::size_t{1}, std::multiplies()); - } - - std::size_t get_index(size_t i) const - { - std::size_t result = 0; - std::size_t s = 1; - - for(auto k : migraphx::reverse(migraphx::range(m_lens.size()))) - { - std::size_t stride = m_strides[k]; - std::size_t len = m_lens[k]; - std::size_t idx = (i % (s * len)) / s; - result += stride * idx; - s *= len; - } - return result; - } - - std::vector min_lens() const - { - std::vector ret(m_dyn_dims.size()); - std::transform(m_dyn_dims.cbegin(), - m_dyn_dims.cend(), - ret.begin(), - [](const shape::dynamic_dimension& x) { return x.min; }); - return ret; - } - - std::vector max_lens() const - { - std::vector ret(m_dyn_dims.size()); - std::transform(m_dyn_dims.cbegin(), - m_dyn_dims.cend(), - ret.begin(), - [](const shape::dynamic_dimension& x) { return x.max; }); - return ret; - } - - std::vector> opt_lens() const - { - std::vector> ret(m_dyn_dims.size()); - std::transform(m_dyn_dims.cbegin(), - m_dyn_dims.cend(), - ret.begin(), - [](const shape::dynamic_dimension& x) { return x.optimals; }); - return ret; - } - - // Does the shape skip over elements? - bool skips() const - { - assert(m_lens.size() == m_strides.size()); - if(elements() == 1) - return false; - return std::none_of(m_strides.begin(), m_strides.end(), [](auto x) { return x == 1; }); - } - - std::shared_ptr copy() const { return std::make_shared(*this); } -}; - -std::string shape::to_sizes_string(const std::vector& shapes) -{ - std::vector sizes; - std::transform(shapes.begin(), shapes.end(), std::back_inserter(sizes), [&](const shape& s) { - std::string r = to_string_range(s.lens(), "x"); - if(not s.standard()) - r += ":" + to_string_range(s.strides(), "x"); - return r; - }); - return join_strings(sizes, ", "); -} - -const std::vector& shape::types() -{ - static const std::vector result = { -#define MIGRAPHX_GENERATE_TYPE_VECTOR(x, t) x, - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_GENERATE_TYPE_VECTOR) tuple_type}; - return result; -} - -std::string shape::name(shape::type_t t) -{ - switch(t) - { - case tuple_type: return "tuple_type"; -#define MIGRAPHX_SHAPE_GENERATE_TYPE_NAME_CASE(x, t) \ - case x: return #x; - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_TYPE_NAME_CASE) -#undef MIGRAPHX_SHAPE_GENERATE_TYPE_NAME_CASE - } - MIGRAPHX_THROW("Invalid type"); -} - -std::string shape::cpp_type(shape::type_t t) -{ - switch(t) - { - case tuple_type: MIGRAPHX_THROW("No C++ type for tuple"); -#define MIGRAPHX_SHAPE_GENERATE_CPP_TYPE_CASE(x, t) \ - case x: return #t; - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_CPP_TYPE_CASE) -#undef MIGRAPHX_SHAPE_GENERATE_CPP_TYPE_CASE - } - MIGRAPHX_THROW("Invalid type"); -} - -bool shape::is_integral(shape::type_t t) -{ - bool result = false; - visit(t, [&](auto as) { result = as.is_integral(); }); - return result; -} - -bool shape::is_compatible(const shape& actual, const shape& expected) -{ - // Check subshapes - if(expected.type() == shape::tuple_type) - return migraphx::equal(actual.sub_shapes(), expected.sub_shapes(), &is_compatible); - if(actual == expected) - return true; - if(actual.type() != expected.type()) - return false; - // Only the expected can be dynamic - if(expected.dynamic()) - return actual.ndim() == expected.ndim(); - if(actual.dynamic()) - return false; - if(actual.lens() != expected.lens()) - return false; - // Check strides from dimensions that are not 1 - return all_of(range(actual.lens().size()), [&](auto i) { - if(actual.lens()[i] == 1) - return true; - return actual.strides()[i] == expected.strides()[i]; - }); -} - -bool shape::is_unsigned(shape::type_t t) -{ - bool result = false; - visit(t, [&](auto as) { result = as.is_unsigned(); }); - return result; -} - -shape::shape() : impl(shape_impl::default_shape()) {} - -shape::shape(type_t t) : impl(std::make_shared(t)) {} - -shape::shape(type_t t, std::vector l) - : impl(std::make_shared(t, std::move(l))) -{ -} - -shape::shape(type_t t, std::vector l, std::vector s) - : impl(std::make_shared(t, std::move(l), std::move(s))) -{ -} - -shape::shape(type_t t, std::initializer_list d) - : shape::shape(t, std::vector{d.begin(), d.end()}) -{ -} - -shape::shape(type_t t, std::vector dims) - : impl(std::make_shared(t, std::move(dims))) -{ -} - -shape::shape(type_t t, - std::vector mins, - std::vector maxes, - std::vector> optimals_list) - : impl(std::make_shared( - t, std::move(mins), std::move(maxes), std::move(optimals_list))) -{ -} - -shape::shape(const std::vector& subs) : impl(std::make_shared(subs)) {} - -shape::shape(std::shared_ptr pimpl) : impl(std::move(pimpl)) {} - -shape shape::from_permutation(type_t t, - const std::vector& l, - const std::vector& perm) -{ - auto new_lens = reorder_dims(l, perm); - shape result = reorder_shape({t, new_lens}, invert_permutation(perm)); - assert(result.lens() == l); - return result; -} - -shape::type_t shape::type() const { return impl->m_type; } - -const std::vector& shape::lens() const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: lens() called on a dynamic shape"); - } - return impl->m_lens; -} - -const std::vector& shape::strides() const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: strides() called on a dynamic shape"); - } - return impl->m_strides; -} - -std::size_t shape::ndim() const -{ - if(this->dynamic()) - { - return dyn_dims().size(); - } - return lens().size(); -} - -std::size_t shape::elements() const { return impl->elements(); } - -std::size_t shape::bytes() const -{ - if(this->sub_shapes().empty()) - { - std::size_t n = 0; - this->visit_type([&](auto as) { n = as.size(); }); - return n * this->element_space(); - } - else - { - return std::accumulate(this->sub_shapes().begin(), - this->sub_shapes().end(), - std::size_t{0}, - [&](auto x, auto y) { return x + y.bytes(); }); - } -} - -std::size_t shape::type_size() const -{ - std::size_t n = 0; - if(this->sub_shapes().empty()) - this->visit_type([&](auto as) { n = as.size(); }); - return n; -} - -std::size_t shape::index(std::initializer_list l) const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: index() called on dynamic shape"); - } - assert(l.size() <= this->lens().size()); - assert(this->lens().size() == this->strides().size()); - return std::inner_product(l.begin(), l.end(), this->strides().begin(), std::size_t{0}); -} - -std::size_t shape::index(const std::vector& l) const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: index() called on dynamic shape"); - } - assert(l.size() <= this->lens().size()); - assert(this->lens().size() == this->strides().size()); - return std::inner_product(l.begin(), l.end(), this->strides().begin(), std::size_t{0}); -} - -std::size_t shape::index(std::size_t i) const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: index() called on dynamic shape"); - } - assert(this->lens().size() == this->strides().size()); - if(this->standard()) - return i; - - return impl->get_index(i); -} - -std::vector shape::multi(std::size_t idx) const -{ - assert(idx < elements()); - std::vector indices(lens().size()); - multi_copy(idx, indices.data(), indices.data() + lens().size()); - return indices; -} - -void shape::multi_copy(std::size_t idx, std::size_t* start, const std::size_t* end) const -{ - size_t tidx = idx; - (void)end; - assert(idx < elements()); - assert(lens().size() <= (end - start)); - for(size_t ii = lens().size() - 1; ii > 0; ii--) - { - *(start + ii) = tidx % lens()[ii]; - tidx = tidx / lens()[ii]; - } - *start = tidx; -} - -bool shape::multi_within_bounds(std::vector multi) const -{ - assert(this->lens().size() == multi.size()); - return std::equal(multi.begin(), multi.end(), this->lens().begin(), std::less<>{}); -} - -bool shape::packed() const -{ - if(this->dynamic()) - { - return false; - } - return this->sub_shapes().empty() and not impl->skips() and - this->elements() == this->element_space(); -} - -bool shape::transposed() const -{ - if(this->dynamic()) - { - return false; - } - if(this->broadcasted()) - { - // TODO: Use a filter_iterator instead - std::vector s; - s.reserve(this->strides().size()); - std::copy_if(this->strides().begin(), - this->strides().end(), - std::back_inserter(s), - [](std::size_t x) { return x != 0; }); - return not std::is_sorted(s.rbegin(), s.rend()); - } - else - { - return not std::is_sorted(this->strides().rbegin(), this->strides().rend()); - } -} - -bool shape::broadcasted() const -{ - if(this->dynamic()) - { - return false; - } - assert(this->lens().size() == this->strides().size()); - return std::any_of( - this->strides().begin(), this->strides().end(), [](auto x) { return x == 0; }); -} - -bool shape::scalar() const -{ - if(this->dynamic()) - { - return false; - } - assert(this->lens().size() == this->strides().size()); - // if any stride > 0, then accumulate will return false - return this->sub_shapes().empty() and - std::accumulate(this->strides().begin(), this->strides().end(), std::size_t(0)) == 0; -} - -bool shape::standard() const { return impl->m_standard; } - -shape shape::normalize_standard() const -{ - if(this->standard()) - return {this->type(), this->lens()}; - else - return *this; -} - -shape shape::as_standard() const -{ - if(not this->dynamic()) - return {this->type(), this->lens()}; - else - return *this; -} - -shape shape::with_lens(type_t t, const std::vector& l) const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: with_lens() called on dynamic shape"); - } - assert(l.size() == this->lens().size()); - auto perm = find_permutation(*this); - return shape::from_permutation(t, l, perm); -} - -shape shape::with_lens(const std::vector& l) const -{ - if(this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: with_lens() called on dynamic shape"); - } - return this->with_lens(this->type(), l); -} - -shape shape::with_type(type_t t) const -{ - auto c = impl->copy(); - c->m_type = t; - return {c}; -} - -shape shape::to_dynamic() const -{ - if(not sub_shapes().empty()) - { - std::vector subs; - std::transform(sub_shapes().cbegin(), - sub_shapes().cend(), - std::back_inserter(subs), - [](auto s) { return s.to_dynamic(); }); - return shape(subs); - } - if(this->dynamic()) - { - return *this; - } - return {type(), lens(), lens(), {}}; -} - -shape shape::to_static(std::size_t x) const -{ - if(not sub_shapes().empty()) - { - std::vector subs; - std::transform(sub_shapes().cbegin(), - sub_shapes().cend(), - std::back_inserter(subs), - [&](auto s) { return s.to_static(x); }); - return shape(subs); - } - if(not this->dynamic()) - { - return *this; - } - auto static_lens = this->max_lens(); - std::transform(static_lens.begin(), - static_lens.end(), - this->dyn_dims().cbegin(), - static_lens.begin(), - [&](auto sl, auto dd) { return dd.is_fixed() ? sl : x; }); - return {type(), static_lens}; -} - -std::size_t shape::element_space() const { return impl->element_space(); } - -std::string shape::type_string() const { return name(this->type()); } - -bool shape::dynamic() const { return not impl->m_dyn_dims.empty(); } - -bool shape::any_of_dynamic() const -{ - if(this->dynamic()) - { - return true; - } - return std::any_of(this->sub_shapes().cbegin(), this->sub_shapes().cend(), [](auto s) { - return s.any_of_dynamic(); - }); -} - -const std::vector& shape::dyn_dims() const -{ - if(not this->dynamic()) - { - MIGRAPHX_THROW("SHAPE: dyn_dims() called on a static shape"); - } - return impl->m_dyn_dims; -} - -std::vector shape::min_lens() const -{ - return this->dynamic() ? impl->min_lens() : this->lens(); -} - -std::vector shape::max_lens() const -{ - return this->dynamic() ? impl->max_lens() : this->lens(); -} - -std::vector> shape::opt_lens() const { return impl->opt_lens(); } - -bool shape::dynamic_dimension::is_fixed() const { return this->min == this->max; } - -bool shape::dynamic_dimension::has_optimal() const { return not optimals.empty(); } - -shape::dynamic_dimension& shape::dynamic_dimension::operator+=(const std::size_t& x) -{ - this->min += x; - this->max += x; - std::set new_optimals; - std::transform(this->optimals.begin(), - this->optimals.end(), - std::inserter(new_optimals, new_optimals.begin()), - [&x](const auto& opt) { return (opt + x); }); - this->optimals = new_optimals; - return *this; -} - -shape::dynamic_dimension& shape::dynamic_dimension::operator-=(const std::size_t& x) -{ - assert(this->min >= x); - assert(this->max >= x); - this->min -= x; - this->max -= x; - std::set new_optimals; - std::transform(this->optimals.begin(), - this->optimals.end(), - std::inserter(new_optimals, new_optimals.begin()), - [&x](const auto& opt) { - assert(opt >= x); - return (opt - x); - }); - this->optimals = new_optimals; - return *this; -} - -bool operator==(const shape::dynamic_dimension& x, const shape::dynamic_dimension& y) -{ - // don't check optimals if both are fixed - return (x.min == y.min and x.max == y.max and - ((x.is_fixed() and y.is_fixed()) or (x.optimals == y.optimals))); -} - -bool operator!=(const shape::dynamic_dimension& x, const shape::dynamic_dimension& y) -{ - return not(x == y); -} -std::ostream& operator<<(std::ostream& os, const shape::dynamic_dimension& x) -{ - os << "[ " << x.min << ", " << x.max << ", {" << migraphx::to_string_range(x.optimals) << "} ]"; - return os; -} - -bool operator==(const shape::dynamic_dimension& x, const std::size_t& y) -{ - return x.min == y and x.max == y; -} -bool operator==(const std::size_t& x, const shape::dynamic_dimension& y) { return y == x; } -bool operator!=(const shape::dynamic_dimension& x, const std::size_t& y) { return not(x == y); } -bool operator!=(const std::size_t& x, const shape::dynamic_dimension& y) { return not(x == y); } - -shape::dynamic_dimension operator+(const shape::dynamic_dimension& x, const std::size_t& y) -{ - auto dd = x; - return dd += y; -} - -shape::dynamic_dimension operator+(const std::size_t& x, const shape::dynamic_dimension& y) -{ - return y + x; -} - -shape::dynamic_dimension operator-(const shape::dynamic_dimension& x, const std::size_t& y) -{ - auto dd = x; - return dd -= y; -} - -bool operator==(const shape& x, const shape& y) -{ - if(x.dynamic() and y.dynamic()) - { - return x.impl == y.impl or (x.type() == y.type() and x.dyn_dims() == y.dyn_dims() and - x.sub_shapes() == y.sub_shapes()); - } - return x.impl == y.impl or - (x.dynamic() == y.dynamic() and x.type() == y.type() and x.lens() == y.lens() and - x.strides() == y.strides() and x.sub_shapes() == y.sub_shapes()); -} - -bool operator!=(const shape& x, const shape& y) { return not(x == y); } - -std::ostream& operator<<(std::ostream& os, const shape& x) -{ - if(x.sub_shapes().empty()) - { - if(x.dynamic()) - { - os << "dynamic, "; - os << x.type_string() << ", "; - os << "{" << to_string_range(x.dyn_dims()) << "}"; - } - else - { - os << x.type_string() << ", "; - os << "{" << to_string_range(x.lens()) << "}, "; - os << "{" << to_string_range(x.strides()) << "}"; - } - } - else - { - os << "[" << to_string_range(x.sub_shapes()) << "]"; - } - return os; -} - -shape::type_t shape::parse_type(const std::string& s) -{ - static const std::unordered_map m = { -#define MIGRAPHX_SHAPE_GENERATE_TYPE_STRING_MAP(x, t) {#x, x}, {#t, x}, - MIGRAPHX_SHAPE_VISIT_TYPES(MIGRAPHX_SHAPE_GENERATE_TYPE_STRING_MAP){"tuple_type", - tuple_type}, - {"tuple", tuple_type}}; - return m.at(s); -} - -const std::vector& shape::sub_shapes() const { return impl->m_shapes; } - -std::vector flatten(const std::vector& shapes) -{ - std::vector result; - for(const auto& s : shapes) - { - if(s.type() == shape::tuple_type) - { - auto subs = flatten(s.sub_shapes()); - result.insert(result.end(), subs.begin(), subs.end()); - } - else - { - result.push_back(s); - } - } - return result; -} - -void migraphx_to_value(value& v, const shape& s) -{ - value result; - result["type"] = migraphx::to_value(s.type_string()); - result["sub_shapes"] = migraphx::to_value(s.sub_shapes()); - // avoid calling functions that will throw - if(s.dynamic()) - { - result["lens"] = {}; - result["strides"] = {}; - result["dynamic_dimensions"] = migraphx::to_value(s.dyn_dims()); - } - else - { - result["lens"] = migraphx::to_value(s.lens()); - result["strides"] = migraphx::to_value(s.strides()); - result["dynamic_dimensions"] = {}; - } - v = result; -} - -void migraphx_from_value(const value& v, shape& s) -{ - auto t = v.at("type").get_string(); - if(t == "tuple_type") - { - s = shape{migraphx::from_value>(v.at("sub_shapes"))}; - } - else - { - if(v.at("dynamic_dimensions").empty()) - { - s = shape{shape::parse_type(t), - v.at("lens").to_vector(), - v.at("strides").to_vector()}; - } - else - { - auto v_dd = v.at("dynamic_dimensions"); - std::vector dyn_dims(v.at("dynamic_dimensions").size()); - std::transform( - v_dd.begin(), v_dd.end(), dyn_dims.begin(), [](const migraphx::value& x) { - return from_value(x); - }); - - s = shape{shape::parse_type(t), dyn_dims}; - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/shape_transform_descriptor.cpp b/docker/rocm/migraphx/shape_transform_descriptor.cpp deleted file mode 100644 index 987cc8915..000000000 --- a/docker/rocm/migraphx/shape_transform_descriptor.cpp +++ /dev/null @@ -1,1263 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using dimension = shape_transform_descriptor::dimension; - -template -static auto compute_end_dim(Iterator start, Iterator last, std::size_t dim, Projection proj) -{ - std::size_t x = 1; - auto it = std::find_if(start, last, [&](auto d) { - x *= proj(d); - return x == dim; - }); - if(it != last) - return it; - return start; -} - -void debug_print(const std::vector& subs) -{ - std::cout << '[' << stream_range(subs) << "]\n"; -} -void debug_print(const dimension& dim) { debug_print(dim.subdimensions); } -void debug_print(const std::vector& dims) -{ - stream_write_value(std::cout, dims); - std::cout << std::endl; -} - -shape_transform_descriptor::shape_transform_descriptor(const std::vector& dims) - : rank(dims.size()) -{ - transform(dims, - range(dims.size()), - std::back_inserter(dimensions), - [](std::size_t d, std::size_t a) -> dimension { - return {{dimension::sub{d, {a}}}}; - }); -} - -static std::vector get_all_subdimensions(const std::vector& dimensions) -{ - std::vector result; - for(const auto& dim : dimensions) - { - result.insert(result.end(), dim.subdimensions.begin(), dim.subdimensions.end()); - } - return result; -} - -template -static void for_each_subdimension(Dimensions&& dimensions, Range&& r, F f) -{ - auto start = r.begin(); - auto last = r.end(); - for(auto& dim : dimensions) - { - for(auto& s : dim.subdimensions) - { - if(start == last) - return; - f(s, *start); - start++; - } - } -} - -// Group all axes into a map with a key of the axis and the value is vector of -// all subdimensions that have that axis. -static std::map> -group_axes(std::vector& dimensions) -{ - std::map> axes_map; - for(auto& d : dimensions) - { - for(auto& s : d.subdimensions) - { - if(s.origin_axis().empty()) - continue; - axes_map[s.origin_axis().front()].push_back(&s); - } - } - return axes_map; -} - -std::vector compute_dims(const operation& op, const std::vector& idims) -{ - shape s{shape::float_type, idims}; - return op.compute_shape({s}).lens(); -} - -std::vector compute_dims(const std::vector& ops, - const std::vector& idims) -{ - shape s{shape::float_type, idims}; - for(const auto& op : ops) - s = op.compute_shape({s}); - return s.lens(); -} - -shape_transform_descriptor shape_transform_descriptor::create(const std::vector& dims, - const std::vector& ops) -{ - shape_transform_descriptor result{dims}; - if(not result.apply(ops)) - return {}; - result.simplify(); - assert(compute_dims(ops, dims) == compute_dims(result.generate(), dims)); - return result; -} - -shape_transform_descriptor -shape_transform_descriptor::rebase(const std::vector& dims) const -{ - auto result = *this; - auto axes_map = group_axes(result.dimensions); - for(auto& [axis, subs] : axes_map) - { - assert(axis < dims.size()); - auto dim = dims[axis]; - auto final_dim = transform_accumulate(subs.begin(), - subs.end(), - std::size_t{1}, - std::multiplies<>{}, - [](const dimension::sub* s) { return s->len; }); - if(dim == final_dim) - { - for(auto* sub : subs) - sub->expose(); - } - else if(dim == 1) - { - for(auto* sub : subs) - { - if(not sub->has_hidden_axis()) - sub->len = 1; - } - } - else if(subs.size() == 1) - { - subs.front()->len = dim; - subs.front()->expose(); - } - else - MIGRAPHX_THROW("Invalid rebase"); - } - result.simplify(); - - return result; -} -static dimension::sub* get_last_subdimension(std::vector& dims) -{ - if(dims.empty()) - return {}; - auto& d = dims.back(); - if(d.subdimensions.empty()) - return nullptr; - return &d.subdimensions.back(); -} - -bool shape_transform_descriptor::apply(const std::vector& ops) -{ - std::vector dims; - std::transform(dimensions.begin(), - dimensions.end(), - std::back_inserter(dims), - [](const dimension& d) { return d.len(); }); - for(const auto& op : ops) - { - auto v = op.to_value(); - if(contains({"reshape", "squeeze", "unsqueeze", "flatten"}, op.name())) - { - dims = compute_dims(op, dims); - if(not apply_reshape(dims)) - return false; - } - else if(op.name() == "transpose") - { - dims = compute_dims(op, dims); - if(not apply_transpose(v["permutation"].to_vector())) - return false; - } - else if(op.name() == "multibroadcast") - { - dims = compute_dims(op, dims); - // cppcheck-suppress knownConditionTrueFalse - if(not apply_broadcast(dims)) - return false; - } - else if(op.name() == "broadcast") - { - dims = compute_dims(op, dims); - // cppcheck-suppress knownConditionTrueFalse - if(not apply_broadcast(dims, v["axis"].to())) - return false; - } - else if(op.name() != "contiguous") - { - return false; - } - } - return true; -} -bool shape_transform_descriptor::apply_reshape(const std::vector& rdims) -{ - std::vector idims; - transform(get_all_subdimensions(dimensions), - std::back_inserter(idims), - std::mem_fn(&dimension::sub::len)); - auto cdims = common_dims::compute(idims, rdims).dims; - if(not cdims.empty() and not apply_reshape_impl(cdims)) - return false; - return apply_reshape_impl(rdims); -} -bool shape_transform_descriptor::apply_reshape_impl(const std::vector& rdims) -{ - assert(migraphx::elements(rdims) == this->elements()); - if(migraphx::equal( - dimensions, rdims, [](const dimension& d, std::size_t rdim) { return d.len() == rdim; })) - return true; - std::vector new_dims; - auto subs = get_all_subdimensions(dimensions); - std::size_t i = 0; - std::size_t r = 0; - while(i < subs.size() and r < rdims.size()) - { - const auto& sub = subs[i]; - auto idim = sub.len; - auto rdim = rdims[r]; - if(idim == rdim) - { - new_dims.push_back({{sub}}); - } - // squeeze - else if(rdim > idim) - { - auto start = subs.begin() + i; - auto it = compute_end_dim(start, subs.end(), rdim, std::mem_fn(&dimension::sub::len)); - if(it == start) - return false; - assert(it != subs.end()); - auto n = it - start; - i += n; - new_dims.push_back({{start, it + 1}}); - } - // unsqueeze - else // if(rdim < idim) - { - auto start = rdims.begin() + r; - auto it = compute_end_dim(start, rdims.end(), idim, id{}); - if(it == start) - return false; - assert(it != rdims.end()); - auto n = it - start; - r += n; - transform(range(n + 1), std::back_inserter(new_dims), [&](auto j) -> dimension { - auto new_sub = sub; - new_sub.add_split_axis(j); - new_sub.len = start[j]; - return {{new_sub}}; - }); - } - r++; - i++; - } - - // Handle trailing 1s - if(new_dims.size() < rdims.size() and not new_dims.empty()) - { - auto* sub = get_last_subdimension(new_dims); - auto axis = sub == nullptr ? std::vector{} : sub->axis; - auto trailing_dims = range(rdims.begin() + new_dims.size(), rdims.end()); - if(any_of(trailing_dims, [](auto d) { return d != 1; })) - return false; - if(distance(trailing_dims) > 1) - sub->add_split_axis(0); - transform(range(distance(trailing_dims)), - std::back_inserter(new_dims), - [&](std::size_t j) -> dimension { - dimension::sub s{1, axis}; - s.add_split_axis(j + 1); - return {{s}}; - }); - } - assert(rdims.size() == new_dims.size()); - if(rdims.size() != new_dims.size()) - return false; - dimensions = new_dims; - return true; -} - -bool shape_transform_descriptor::apply_transpose(const std::vector& permutation) -{ - if(permutation.size() != dimensions.size()) - return false; - dimensions = reorder_dims(dimensions, permutation); - return true; -} - -bool shape_transform_descriptor::apply_broadcast(const std::vector& out_lens, - optional axis) -{ - auto offset = axis.value_or(out_lens.size() - dimensions.size()); - - std::vector new_dims; - std::transform(out_lens.begin(), - out_lens.begin() + offset, - std::back_inserter(new_dims), - [&](auto len) -> dimension { - return {{dimension::sub{len, {}}}}; - }); - std::transform(dimensions.begin(), - dimensions.end(), - out_lens.begin() + offset, - std::back_inserter(new_dims), - [&](const dimension& dim, auto len) -> dimension { - if(len == dim.len()) - return dim; - if(dim.len() != 1) - MIGRAPHX_THROW("Wrong out_lens for broadcast"); - auto new_subs = dim.subdimensions; - if(not new_subs.empty()) - { - new_subs.front().len = len; - } - for(auto& s : new_subs) - { - s.hide(); - } - return {new_subs}; - }); - std::transform(out_lens.begin() + offset + dimensions.size(), - out_lens.end(), - std::back_inserter(new_dims), - [&](auto len) -> dimension { - return {{dimension::sub{len, {}}}}; - }); - assert(out_lens.size() == new_dims.size()); - dimensions = new_dims; - return true; -} - -// Remove subdimensions of 1 -void remove_1_sub_dims(std::vector& subdimensions) -{ - subdimensions.erase(std::remove_if(subdimensions.begin(), - subdimensions.end(), - [&](const dimension::sub& d) { return d.len == 1; }), - subdimensions.end()); -} - -void dimension::simplify() -{ - if(subdimensions.size() < 2) - return; - remove_1_sub_dims(subdimensions); - // Flatten adjacent dimensions - adjacent_for_each(subdimensions.begin(), subdimensions.end(), [&](sub& d1, sub& d2) { - if(d1.origin_axis().size() < 2) - return; - if(d2.origin_axis().size() < 2) - return; - if(d1.has_hidden_axis() != d2.has_hidden_axis()) - return; - if(not std::equal(d1.origin_axis().begin(), - d1.origin_axis().end() - 1, - d2.origin_axis().begin(), - d2.origin_axis().end() - 1)) - return; - auto a1 = d1.origin_axis().back(); - auto a2 = d2.origin_axis().back(); - assert(a2 != a1); - if(a2 <= a1) - return; - if((a2 - a1) != 1) - return; - d2.len = d1.len * d2.len; - d1.len = 1; - }); - remove_1_sub_dims(subdimensions); -} - -// Search all subdimensions and return the subdimensions vector, an iterator -// to the subdimension found and an optional iterator to the previous -// subdimension if available. -template -static auto find_subdimension(shape_transform_descriptor& td, Predicate p) -{ - dimension* prev_dim = nullptr; - for(auto& d : td.dimensions) - { - auto it = std::find_if(d.subdimensions.begin(), d.subdimensions.end(), p); - if(it != d.subdimensions.end()) - { - decltype(std::make_optional(it)) prev = nullopt; - if(it == d.subdimensions.begin()) - { - if(prev_dim != nullptr and not prev_dim->subdimensions.empty()) - { - prev = std::prev(prev_dim->subdimensions.end()); - } - } - else - { - prev = std::prev(it); - } - return std::make_tuple(&d.subdimensions, it, prev); - } - prev_dim = &d; - } - MIGRAPHX_THROW("Searching for non-existent subdimension"); -} - -static bool is_broadcast_dim(const dimension& d) -{ - if(d.len() == 1) - return false; - assert(not d.subdimensions.empty()); - if(d.subdimensions.size() != 1) - return false; - const auto& sub = d.subdimensions.front(); - return sub.axis.empty(); -} - -static bool missing_leading_axis(const dimension& d) -{ - if(d.subdimensions.empty()) - return true; - const auto& sub = d.subdimensions.front(); - return sub.origin_axis().empty(); -} - -static void set_broadcast_dim(dimension& d, std::size_t axis) -{ - if(d.subdimensions.empty()) - d.subdimensions.push_back({1, {axis}}); - else - { - assert(d.subdimensions.front().hidden_axis.empty()); - d.subdimensions.front().hidden_axis = {axis}; - } -} - -static void set_origin_axis(dimension::sub& s, const std::vector& axis) -{ - if(s.has_hidden_axis()) - s.hidden_axis = axis; - else - s.axis = axis; -} - -// If an axis is split and some dimensions are hidden and others are not, then -// remove the hidden axis so only the non-hidden axis is used in -// simplificaiton -static void remove_split_hidden_axes(std::map>& axes_map) -{ - for(auto&& p : axes_map) - { - auto& subs = p.second; - if(std::all_of(subs.begin(), subs.end(), [](const dimension::sub* s) { - return s->has_hidden_axis(); - })) - continue; - for(auto* sub : subs) - { - if(not sub->has_hidden_axis()) - continue; - sub->hidden_axis.clear(); - } - // Remove the subdimesions that no longer have an axis - subs.erase(std::remove_if(subs.begin(), - subs.end(), - [](const dimension::sub* s) { - return s->axis.empty() and s->hidden_axis.empty(); - }), - subs.end()); - } - // Remove axis from group if empty - erase_if(axes_map, [](auto&& p) { return p.second.empty(); }); -} - -// If this is scalar, then remove all axes -static void remove_scalar_axis(std::vector& dimensions) -{ - dimension::sub* s = nullptr; - for(auto& d : dimensions) - { - auto has_axis = [](const dimension::sub& x) { return not x.origin_axis().empty(); }; - auto it = std::find_if(d.subdimensions.begin(), d.subdimensions.end(), has_axis); - if(it == d.subdimensions.end()) - continue; - if(s != nullptr) - return; - if(std::count_if(std::next(it), d.subdimensions.end(), has_axis) > 0) - return; - s = &*it; - } - if(s != nullptr) - { - if(s->has_hidden_axis()) - s->hidden_axis.clear(); - if(s->len == 1) - s->axis.clear(); - } -} - -// Renumber all axes while preserving the order of the axes -static void renumber_axes(std::map>& axes_map) -{ - for(auto&& p : axes_map) - { - const auto& axis = p.first; - auto& subs = p.second; - if(subs.size() == 1) - { - set_origin_axis(*subs[0], {axis}); - } - else - { - std::sort(subs.begin(), subs.end(), by(std::less<>{}, [](const dimension::sub* s) { - return s->origin_axis(); - })); - for(std::size_t i : range(subs.size())) - set_origin_axis(*subs[i], {axis, i}); - } - } -} -static void renumber_axes(std::vector& dimensions) -{ - auto axes_map = group_axes(dimensions); - renumber_axes(axes_map); -} -static void collapse_1_dims(std::vector& dimensions) -{ - // Find a dimension that ends with a subdimension of 1 with a single axis, - // and is followed by subdimension in the next dimension of 1 that has a - // split axis. It will remove the trailing subdimension and update the - // leading subdimension to use the axis from the trailing subdimension. - adjacent_for_each(dimensions.begin(), dimensions.end(), [&](dimension& d1, dimension& d2) { - if(d1.subdimensions.size() < 2) - return; - if(d2.subdimensions.empty()) - return; - if(d2.len() != 1) - return; - const auto& sub1 = d1.subdimensions.back(); - auto& sub2 = d2.subdimensions.front(); - if(sub1.axis.size() != 1) - return; - if(sub2.axis.size() < 2) - return; - if(sub1.len != 1) - return; - if(sub2.len != 1) - return; - sub2.axis = sub1.axis; - d1.subdimensions.pop_back(); - }); - - renumber_axes(dimensions); -} - -void shape_transform_descriptor::simplify() -{ - for(auto& d : dimensions) - d.simplify(); - - remove_scalar_axis(dimensions); - - std::map missing_axes; - std::vector last_axis; - { - // Group axes - auto axes_map = group_axes(dimensions); - if(axes_map.empty()) - return; - - remove_split_hidden_axes(axes_map); - renumber_axes(axes_map); - - // Find last axis - last_axis = std::prev(axes_map.end())->second.back()->axis; - - // Find missing axes. This will store a mapping between the missing - // axis and the next available axis. - for(auto axis : range(rank)) - { - if(contains(axes_map, axis)) - continue; - auto it = axes_map.upper_bound(axis); - missing_axes[axis] = it == axes_map.end() ? rank : it->first; - } - } - - // Find broadcasted dimensions. This will store a map from the next axis - // to the indices of the previous dimensions that are being broadcasted. - std::map> broadcast_dims_map; - group_find( - dimensions.begin(), dimensions.end(), &missing_leading_axis, [&](auto start, auto last) { - auto axis = rank; - if(last != dimensions.end()) - { - assert(not last->subdimensions.empty()); - const auto& sub = last->subdimensions.front(); - assert(not sub.origin_axis().empty()); - axis = sub.origin_axis().front(); - } - std::deque dims(std::distance(start, last)); - std::iota(dims.begin(), dims.end(), std::distance(dimensions.begin(), start)); - broadcast_dims_map[axis] = dims; - }); - - // Reinsert removed axis of 1. This tries to insert the missing axis next - // to an adjacent axis or used as one of the broadcasted axes in order to - // minimize transposition. - for(auto&& p : missing_axes) - { - auto missing_axis = p.first; - auto next_axis = p.second; - auto missing_sub = dimension::sub{1, {missing_axis}}; - // If next_axis is the rank that means there isnt another axis to - // search for, so instead try to insert the axis at the end. - if(next_axis == rank) - { - auto [sub, it, prev] = find_subdimension( - *this, [&](const dimension::sub& s) { return s.axis == last_axis; }); - // Check if we can insert it at the end - auto bdims = broadcast_dims_map.find(rank); - if(bdims != broadcast_dims_map.end() and not bdims->second.empty()) - { - auto bdim = bdims->second.front(); - bdims->second.pop_front(); - set_broadcast_dim(dimensions[bdim], missing_axis); - } - else - { - auto next = std::find_if(std::next(it), sub->end(), [&](const dimension::sub& s) { - if(s.len != 1) - return true; - if(s.axis.empty()) - return true; - return s.axis.front() > missing_axis; - }); - sub->insert(next, missing_sub); - } - } - else - { - // Search for the subdimension that has the next axis and try to - // insert the axis before it will be in order. - auto [sub, it, prev] = find_subdimension(*this, [&](const dimension::sub& s) { - if(s.origin_axis().empty()) - return false; - if(s.origin_axis().front() != next_axis) - return false; - if(s.origin_axis().size() == 1) - return true; - assert(s.origin_axis().size() == 2); - return s.origin_axis().back() == 0; - }); - bool in_order = false; - if(prev.has_value() and not(*prev)->origin_axis().empty()) - in_order = (*prev)->origin_axis().front() == missing_axis - 1; - // If the axis is not inorder then see if we can find a broadcast axis to place it - auto bdims = - in_order ? broadcast_dims_map.end() : broadcast_dims_map.upper_bound(missing_axis); - if(bdims != broadcast_dims_map.end() and not bdims->second.empty()) - { - auto bdim = bdims->second.front(); - bdims->second.pop_front(); - set_broadcast_dim(dimensions[bdim], missing_axis); - } - else - { - sub->insert(it, missing_sub); - } - } - } - - collapse_1_dims(dimensions); -} - -static std::size_t get_len(const dimension::sub& s, const std::vector& input_dims) -{ - if(input_dims.empty()) - return s.len; - if(s.axis.empty()) - return s.len; - auto dim = input_dims.at(s.axis.front()); - if(dim == 0) - return s.len; - if(dim == 1) - return 1; - if(s.axis.size() == 1) - return dim; - return s.len; -} - -static operation make_reshape_squeeze(const std::vector& new_dims) -{ - // Can use squeeze - if(std::all_of(new_dims.begin(), new_dims.end(), [](const dimension& d) { - if(d.subdimensions.size() < 2) - return true; - auto n = std::count_if(d.subdimensions.begin(), - d.subdimensions.end(), - [&](const dimension::sub& s) { return s.len == 1; }); - return n >= (d.subdimensions.size() - 1); - })) - { - std::vector base_axes = {0}; - transform_partial_sum( - new_dims.begin(), - std::prev(new_dims.end()), - std::back_inserter(base_axes), - std::plus<>{}, - [](const dimension& d) { return std::max(1, d.subdimensions.size()); }); - auto get_squeezed_axes = [](const dimension& d, std::size_t base_axis) { - std::vector result; - if(d.subdimensions.size() < 2) - return result; - auto idx = range(d.subdimensions.size()); - transform_if( - idx.begin(), - idx.end(), - std::back_inserter(result), - [&](std::size_t i) { return d.subdimensions[i].len == 1; }, - [&](std::size_t i) { return base_axis + i; }); - if(result.size() == d.subdimensions.size()) - result.pop_back(); - return result; - }; - std::vector axes; - std::transform(new_dims.begin(), - new_dims.end(), - base_axes.begin(), - join_back_inserter(axes), - get_squeezed_axes); - return make_op("squeeze", {{"axes", axes}}); - } - else - { - std::vector dims; - std::transform(new_dims.begin(), - new_dims.end(), - std::back_inserter(dims), - [](const dimension& d) -> std::size_t { - if(is_broadcast_dim(d)) - return 1; - return d.len(); - }); - return make_op("reshape", {{"dims", dims}}); - } -} - -static void flatten_broadcasted_dim(dimension::sub& s) -{ - if(s.axis.empty()) - { - s.len = 1; - s.expose(); - } -} - -static operation make_reshape_unsqueeze(const std::vector& subs, - const std::vector& input_dims = {}) -{ - bool use_reshape = false; - std::unordered_set all_1s; - // Check if split dimensions are all additional 1s - if(std::any_of( - subs.begin(), subs.end(), [](const dimension::sub& s) { return s.axis.size() > 1; })) - { - auto subs2 = subs; - auto by_axis = by(std::equal_to<>{}, [](const dimension::sub& s) -> int64_t { - if(s.axis.empty()) - return -1; - return s.axis.front(); - }); - group_by( - subs2.begin(), - subs2.end(), - [&](auto start, auto last) { - if(use_reshape) - return; - // Number of elements - auto n = std::distance(start, last); - if(n < 2) - return; - // Number of elements that are 1 - auto n1 = std::count_if(start, last, [&](const dimension::sub& s) { - return get_len(s, input_dims) == 1; - }); - if(n == n1 and not start->axis.empty()) - all_1s.insert(start->axis.front()); - use_reshape |= std::max(0, n - n1 - 1) > 0; - }, - by_axis); - } - if(use_reshape) - { - std::vector dims; - std::transform(subs.begin(), - subs.end(), - std::back_inserter(dims), - [&](const dimension::sub& s) -> std::size_t { - if(s.axis.empty()) - return 1; - return get_len(s, input_dims); - }); - return make_op("reshape", {{"dims", dims}}); - } - else - { - std::vector axes; - for(auto i : range(subs.size())) - { - const auto& sub = subs[i]; - if(sub.axis.size() == 1) - continue; - if(get_len(sub, input_dims) != 1 and not sub.axis.empty()) - continue; - if(not sub.axis.empty() and contains(all_1s, sub.axis.front()) and sub.axis.back() == 0) - continue; - axes.push_back(i); - } - return make_op("unsqueeze", {{"axes", axes}}); - } -} - -namespace { -struct operation_list -{ - std::vector ops; - - void push_back(const operation& op) { ops.push_back(op); } - - std::vector to_vector() && - { - std::reverse(ops.begin(), ops.end()); - return std::move(ops); - } -}; - -} // namespace - -static bool has_no_axes(const dimension& d) -{ - return std::all_of(d.subdimensions.begin(), d.subdimensions.end(), [](const dimension::sub& s) { - return s.axis.empty() and s.hidden_axis.empty(); - }); -} -static bool has_axes(const dimension& d) -{ - return std::any_of(d.subdimensions.begin(), d.subdimensions.end(), [](const dimension::sub& s) { - return not s.axis.empty(); - }); -} - -static void generate_from_subdimensions(operation_list& result, - std::vector subs, - const std::vector& input_dims = {}) -{ - // Need multibroadcast - if(std::any_of(subs.begin(), subs.end(), [&](const dimension::sub& s) { - return s.axis.empty() and get_len(s, input_dims) != 1; - })) - { - std::vector out_lens; - std::transform(subs.begin(), - subs.end(), - std::back_inserter(out_lens), - [&](const dimension::sub& s) { return get_len(s, input_dims); }); - result.push_back(make_op("multibroadcast", {{"out_lens", out_lens}})); - } - - // Flatten broadcasted subdimensions - std::for_each(subs.begin(), subs.end(), &flatten_broadcasted_dim); - - auto tsubs = subs; - // Inject additonal axis to compute transpose permutation better - auto is_empty_axis = [](const auto& s) { return s.axis.empty(); }; - group_find(tsubs.begin(), tsubs.end(), is_empty_axis, [&](auto start, auto last) { - if(start == tsubs.begin()) - return; - auto base = std::prev(start); - auto axis = base->axis; - axis.push_back(0); - std::for_each(start, last, [&](auto& s) { - s.axis = axis; - axis.back()++; - }); - }); - - auto compare_sub = [](auto f) { - return by(f, [](const dimension::sub& s) -> const auto& { return s.axis; }); - }; - // Need transpose - if(not std::is_sorted(tsubs.begin(), tsubs.end(), compare_sub(std::less<>{}))) - { - auto permutation = sort_permutation(tsubs, compare_sub(std::less<>{})); - result.push_back(make_op("transpose", {{"permutation", invert_permutation(permutation)}})); - subs = reorder_dims(subs, permutation); - } - // Need reshape unsqueeze - if(std::any_of( - subs.begin(), subs.end(), [](const dimension::sub& s) { return s.axis.size() != 1; })) - { - result.push_back(make_reshape_unsqueeze(subs, input_dims)); - } -} - -// This will generate the operators to apply the shape transformation that is -// represented by this class. This is the order of operators that will be -// generated if needed: -// -// 1. Reshape/unsqueeze -// 2. Transpose -// 3. Broadcast -// 4. Reshape/squeeze -// 5. Broadcast -// -// This will generate operators backwards starting at 5 and going up. Steps 1-3 -// are generated from the subdimensions and steps 4-5 are generated with the -// dimensions. -std::vector shape_transform_descriptor::generate() const -{ - operation_list result; - std::vector new_dims = dimensions; - // Need broadcast - if(std::any_of(new_dims.begin(), new_dims.end(), &is_broadcast_dim)) - { - std::vector out_lens; - std::transform(new_dims.begin(), - new_dims.end(), - std::back_inserter(out_lens), - [](const dimension& d) { return d.len(); }); - auto startb = std::find_if_not(new_dims.begin(), new_dims.end(), &has_no_axes); - auto trailb = std::find_if_not(startb, new_dims.end(), &has_axes); - auto axis = std::distance(new_dims.begin(), startb); - auto extra_dims = axis + std::distance(trailb, new_dims.end()); - // Use broadcast instead of multibroadcast - if(std::all_of(trailb, new_dims.end(), &has_no_axes) and extra_dims > 0 and - axis < new_dims.size()) - { - result.push_back(make_op("broadcast", {{"axis", axis}, {"out_lens", out_lens}})); - new_dims.erase(trailb, new_dims.end()); - new_dims.erase(new_dims.begin(), new_dims.begin() + axis); - } - else - { - result.push_back(make_op("multibroadcast", {{"out_lens", out_lens}})); - } - } - // If all the dimensions have no axes then there isnt anthing else to do - // so just clear the new_dims - if(std::all_of(new_dims.begin(), new_dims.end(), &has_no_axes)) - new_dims.clear(); - // Flatten broadcasted dimensions - for(auto& d : new_dims) - { - if(d.subdimensions.size() != 1) - continue; - flatten_broadcasted_dim(d.subdimensions.front()); - } - // Need squeeze reshape - if(std::any_of(new_dims.begin(), new_dims.end(), [](const dimension& d) { - if(d.subdimensions.size() != 1) - return true; - return is_broadcast_dim(d); - })) - { - result.push_back(make_reshape_squeeze(new_dims)); - } - - auto subs = get_all_subdimensions(new_dims); - generate_from_subdimensions(result, subs); - return std::move(result).to_vector(); -} - -bool shape_transform_descriptor::has_broadcast() const -{ - return std::any_of(dimensions.begin(), dimensions.end(), [&](const dimension& d) { - return std::any_of(d.subdimensions.begin(), - d.subdimensions.end(), - [&](const dimension::sub& s) { return s.axis.empty() and s.len != 1; }); - }); -} -void shape_transform_descriptor::flatten_broadcast() -{ - for(auto& d : dimensions) - std::for_each(d.subdimensions.begin(), d.subdimensions.end(), &flatten_broadcasted_dim); -} - -std::vector shape_transform_descriptor::generate_common_from_src( - const std::vector& input_dims) const -{ - operation_list result; - auto subs = get_all_subdimensions(dimensions); - generate_from_subdimensions(result, subs, input_dims); - return std::move(result).to_vector(); -} -std::vector shape_transform_descriptor::generate_common_from_dst( - const std::vector& input_dims) const -{ - // Need reshape - if(std::all_of(dimensions.begin(), dimensions.end(), [](const dimension& d) { - return d.subdimensions.size() == 1; - })) - return {}; - std::vector subs; - // Update axes to point to the destination - for(std::size_t i : range(dimensions.size())) - { - const auto& d = dimensions[i]; - std::transform(d.subdimensions.begin(), - d.subdimensions.end(), - range(d.subdimensions.size()).begin(), - std::back_inserter(subs), - [&](dimension::sub s, auto j) { - s.axis = {i}; - if(d.subdimensions.size() > 1) - s.axis.push_back(j); - return s; - }); - } - return {make_reshape_unsqueeze(subs, input_dims)}; -} -std::vector shape_transform_descriptor::generate_dst_from_common( - const std::vector& input_dims) const -{ - std::vector result; - std::vector new_dims = dimensions; - for_each_subdimension(new_dims, input_dims, [&](auto& s, auto dim) { s.len = dim; }); - - // Remove broadcasted dimensions - for(auto& d : new_dims) - { - if(d.subdimensions.size() != 1) - continue; - auto& s = d.subdimensions.front(); - s.expose(); - } - // Need squeeze reshape - if(std::any_of(new_dims.begin(), new_dims.end(), [](const dimension& d) { - if(d.subdimensions.size() != 1) - return true; - return is_broadcast_dim(d); - })) - { - result.push_back(make_reshape_squeeze(new_dims)); - } - return result; -} - -std::vector> shape_transform_descriptor::common_axes_map_from_src() const -{ - std::vector> result; - auto subs = get_all_subdimensions(dimensions); - std::map> axes_map; - for(const auto& s : subs) - { - if(not s.origin_axis().empty()) - axes_map[s.origin_axis().front()].push_back(&s); - } - for(auto&& p : axes_map) - { - std::sort(p.second.begin(), p.second.end(), by(std::less<>{}, [](const dimension::sub* s) { - return s->axis; - })); - } - assert(not axes_map.empty()); - auto max_axis = std::prev(axes_map.end())->first; - result.resize(max_axis + 1); - for(auto&& p : axes_map) - { - assert(p.first < result.size()); - std::transform(p.second.begin(), - p.second.end(), - std::back_inserter(result[p.first]), - [&](const dimension::sub* s) { return s - subs.data(); }); - } - return result; -} -std::vector> shape_transform_descriptor::common_axes_map_from_dst() const -{ - std::vector> result; - std::size_t start = 0; - for(const auto& d : dimensions) - { - auto& v = result.emplace_back(d.subdimensions.size()); - std::iota(v.begin(), v.end(), start); - start += d.subdimensions.size(); - } - return result; -} - -bool shape_transform_descriptor::empty() const { return dimensions.empty(); } - -std::vector shape_transform_descriptor::lens() const -{ - std::vector result; - std::transform(dimensions.begin(), - dimensions.end(), - std::back_inserter(result), - [](const dimension& d) { return d.len(); }); - return result; -} - -std::size_t dimension::len() const -{ - return transform_accumulate(subdimensions.begin(), - subdimensions.end(), - std::size_t{1}, - std::multiplies<>{}, - [](const auto& s) { return s.len; }); -} - -std::size_t shape_transform_descriptor::elements() const -{ - return transform_accumulate(dimensions.begin(), - dimensions.end(), - std::size_t{1}, - std::multiplies<>{}, - [](const auto& s) { return s.len(); }); -} -std::vector -shape_transform_descriptor::common_dims(const std::vector& input_dims) const -{ - std::vector result; - for(const auto& d : dimensions) - { - std::transform(d.subdimensions.begin(), - d.subdimensions.end(), - std::back_inserter(result), - [&](const dimension::sub& s) { return get_len(s, input_dims); }); - } - return result; -} - -const std::vector& shape_transform_descriptor::dimension::sub::origin_axis() const -{ - return axis.empty() ? hidden_axis : axis; -} -bool shape_transform_descriptor::dimension::sub::has_hidden_axis() const -{ - return axis.empty() and not hidden_axis.empty(); -} - -void shape_transform_descriptor::dimension::sub::add_split_axis(std::size_t i) -{ - if(not axis.empty()) - axis.push_back(i); - if(not hidden_axis.empty()) - hidden_axis.push_back(i); -} - -void shape_transform_descriptor::dimension::sub::expose() -{ - if(has_hidden_axis()) - { - axis = hidden_axis; - hidden_axis.clear(); - } -} - -void shape_transform_descriptor::dimension::sub::hide() -{ - if(not has_hidden_axis()) - { - hidden_axis = axis; - axis.clear(); - } -} - -bool operator==(const dimension::sub& x, const dimension::sub& y) -{ - return by(std::equal_to<>{}, - [](const dimension::sub& s) { return std::tie(s.len, s.axis, s.hidden_axis); })(x, y); -} -bool operator!=(const dimension::sub& x, const dimension::sub& y) { return not(x == y); } -std::ostream& operator<<(std::ostream& os, const dimension::sub& x) -{ - os << x.len << ":" << to_string_range(x.axis, "x"); - if(not x.hidden_axis.empty()) - os << "$" << to_string_range(x.hidden_axis, "x"); - return os; -} -bool operator==(const dimension& x, const dimension& y) -{ - return x.subdimensions == y.subdimensions; -} -bool operator!=(const dimension& x, const dimension& y) { return not(x == y); } -std::ostream& operator<<(std::ostream& os, const dimension& x) -{ - os << '[' << stream_range(x.subdimensions) << ']'; - return os; -} -bool operator==(const shape_transform_descriptor& x, const shape_transform_descriptor& y) -{ - return by(std::equal_to<>{}, [](const shape_transform_descriptor& sd) { - return std::tie(sd.dimensions, sd.rank); - })(x, y); -} -bool operator!=(const shape_transform_descriptor& x, const shape_transform_descriptor& y) -{ - return not(x == y); -} -std::ostream& operator<<(std::ostream& os, const shape_transform_descriptor& x) -{ - stream_write_value(os, x.dimensions); - return os; -} - -std::vector optimize_shape_transforms(const std::vector& dims, - const std::vector& ops) -{ - auto sd = shape_transform_descriptor::create(dims, ops); - if(sd.empty()) - return ops; - return sd.generate(); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/simplify_algebra.cpp b/docker/rocm/migraphx/simplify_algebra.cpp deleted file mode 100644 index e79d4d486..000000000 --- a/docker/rocm/migraphx/simplify_algebra.cpp +++ /dev/null @@ -1,2035 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -auto lit_broadcast() { return match::any_of(match::is_constant(), match::name("broadcast")); } -auto not_lit_broadcast() { return match::none_of(match::is_constant(), match::name("broadcast")); } -auto op_lit_broadcast(std::string op, std::string x, std::string y) -{ - return match::name(std::move(op))(match::either_arg(0, 1)( - lit_broadcast().bind(std::move(x)), not_lit_broadcast().bind(std::move(y)))); -} - -auto conv_const_weights() -{ - return match::name("convolution")( - match::used_once(), - match::args(match::none_of(match::is_constant()), match::is_constant().bind("w"))); -} - -auto from_int4() -{ - return match::make_predicate_matcher([](instruction_ref start) { - return fix([&](auto self, instruction_ref ins) { - auto alias = instruction::get_output_alias(ins); - if(contains({"reshape", "dequantizelinear"}, alias->name())) - return self(alias->inputs().front()); - if(alias->name() == "concat") - return all_of(alias->inputs(), self); - return alias->name() == "unpack_int4"; - })(start); - }); -} - -auto not_from_int4() { return match::none_of(from_int4()); } - -auto reduction() { return match::name_contains("reduce"); } - -// conv(x, w) * a => conv(x, a * w) -struct find_mul_conv -{ - auto matcher() const - { - return match::name("mul")( - match::either_arg(0, 1)(conv_const_weights().bind("conv"), - match::name("broadcast", "multibroadcast").bind("a"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto conv_ins = r.instructions["conv"]; - auto a_ins = r.instructions["a"]; - auto w_ins = r.instructions["w"]; - - const auto& a_input_lens = a_ins->inputs().front()->get_shape().lens(); - - std::size_t num_not_one_dims = std::count_if( - a_input_lens.cbegin(), a_input_lens.cend(), [](auto dim) { return dim != 1; }); - if(num_not_one_dims > 1) - return; - - // check broadcasted along channels - const auto& a_lens = a_ins->get_shape().lens(); - const auto& a_strides = a_ins->get_shape().strides(); - - auto is_broadcasted_axis = [](auto len, auto stride) { return len == 1 or stride == 0; }; - - if(a_strides.at(1) != 1) - return; - - if(not is_broadcasted_axis(a_lens.front(), a_strides.front())) - return; - - if(not std::equal(a_lens.begin() + 2, - a_lens.end(), - a_strides.begin() + 2, - a_strides.end(), - is_broadcasted_axis)) - return; - - auto sq = m.insert_instruction(ins, make_op("squeeze"), a_ins->inputs().front()); - auto new_a = m.insert_instruction( - ins, make_op("broadcast", {{"axis", 0}, {"out_lens", w_ins->get_shape().lens()}}), sq); - auto new_mul = m.insert_instruction(ins, make_op("mul"), new_a, w_ins); - auto new_conv = m.insert_instruction( - ins, conv_ins->get_operator(), conv_ins->inputs().front(), new_mul); - m.replace_instruction(ins, new_conv); - } -}; - -struct find_mul_slice_conv -{ - static auto conv() - { - return match::name("convolution")( - match::all_of[match::outputs()](match::name("slice")), - match::args(match::any(), match::is_constant().bind("w"))); - } - auto matcher() const - { - return match::name("mul")(match::either_arg(0, 1)( - match::name("slice")(match::used_once(), match::arg(0)(conv().bind("conv"))) - .bind("slice"), - match::name("broadcast")(match::is_constant()).bind("a"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto slice_ins = r.instructions["slice"]; - auto conv_ins = r.instructions["conv"]; - auto a_ins = r.instructions["a"]; - auto w_ins = r.instructions["w"]; - - auto broadcast_op = any_cast(a_ins->get_operator()); - if(broadcast_op.axis != 1) - return; - - auto slice_op = any_cast(slice_ins->get_operator()); - if(slice_op.axes.size() != 1) - return; - if(slice_op.axes.front() != 1) - return; - - auto slice_idx = std::distance(conv_ins, slice_ins); - if(std::any_of(conv_ins->outputs().begin(), conv_ins->outputs().end(), [&](auto i) { - if(i == slice_ins) - return false; - if(std::distance(conv_ins, i) < slice_idx) - return true; - auto sop = any_cast(i->get_operator()); - if(sop.axes != slice_op.axes) - return true; - if(std::max(sop.starts.front(), slice_op.starts.front()) < - std::min(sop.ends.front(), slice_op.ends.front())) - return true; - return false; - })) - return; - - auto w_slice_op = slice_op; - w_slice_op.axes = {0}; - auto slice_w_ins = m.insert_instruction(ins, w_slice_op, w_ins); - - auto new_a = m.insert_instruction( - ins, - make_op("broadcast", {{"axis", 0}, {"out_lens", slice_w_ins->get_shape().lens()}}), - a_ins->inputs().front()); - auto new_mul = m.insert_instruction(ins, make_op("mul"), new_a, slice_w_ins); - - std::vector sliced_weights; - if(slice_op.starts.front() != 0) - sliced_weights.push_back(m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", slice_op.starts}}), - w_ins)); - sliced_weights.push_back(new_mul); - int64_t end_axis = w_ins->get_shape().lens().at(0); - if(slice_op.ends.front() != end_axis) - sliced_weights.push_back(m.insert_instruction( - ins, - make_op("slice", {{"axes", {0}}, {"starts", slice_op.ends}, {"ends", {end_axis}}}), - w_ins)); - - auto new_weights = - m.insert_instruction(ins, make_op("concat", {{"axis", 0}}), sliced_weights); - - auto new_conv = m.insert_instruction( - ins, conv_ins->get_operator(), conv_ins->inputs().front(), new_weights); - assert(conv_ins->get_shape() == new_conv->get_shape()); - - auto slice1 = m.insert_instruction(ins, slice_op, new_conv); - assert(ins->get_shape().lens() == slice1->get_shape().lens()); - m.replace_instruction(ins, slice1); - // TODO: Check each slice doesn't overlap and that it occurs after slice_ins - auto outputs = conv_ins->outputs(); - for(auto output : outputs) - if(output != slice_ins) - instruction::replace_argument(output, conv_ins, new_conv); - } -}; - -struct find_mul_dot -{ - auto matcher() const - { - auto constant = match::is_constant(not_from_int4()); - auto is_dot_const_inputs = match::name("dot")(match::any_of[match::inputs()](constant)); - return match::name("mul")(match::either_arg(0, 1)( - is_dot_const_inputs.bind("dot"), match::name("broadcast", "multibroadcast").bind("c"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto dot_ins = r.instructions["dot"]; - auto a_ins = dot_ins->inputs()[0]; - auto b_ins = dot_ins->inputs()[1]; - auto c_ins = r.instructions["c"]; - - const auto& c_strides = c_ins->get_shape().strides(); - - // There should only be one stride that is not zero - if(std::count_if(c_strides.begin(), c_strides.end(), [](auto s) { return s != 0; }) > 1) - return; - - auto add_mul_const = [&](instruction_ref x_ins) { - if(not x_ins->can_eval()) - return m.end(); - auto broadcast_v = c_ins->get_operator().to_value(); - broadcast_v["out_lens"] = x_ins->get_shape().lens(); - - auto cb_ins = - m.insert_instruction(ins, make_op(c_ins->name(), broadcast_v), c_ins->inputs()); - return m.insert_instruction(ins, make_op("mul"), x_ins, cb_ins); - }; - - if(c_strides.back() == 1) - { - b_ins = add_mul_const(b_ins); - } - else if(c_strides[c_strides.size() - 2] == 1) - { - a_ins = add_mul_const(a_ins); - } - else if(c_ins->get_shape().scalar()) - { - if(a_ins->can_eval()) - a_ins = add_mul_const(a_ins); - else - b_ins = add_mul_const(b_ins); - } - else - { - return; - } - - if(contains({a_ins, b_ins}, m.end())) - return; - - m.replace_instruction(ins, make_op("dot"), a_ins, b_ins); - } -}; - -/* -Moves the slice on the output of the Dot operation to slices on the inputs of the Dot operation to -avoid computing redundant values. -e.g. slice(gemm(a, b)) --> gemm(slice(a), slice(b)) -*/ -struct find_dot_slice -{ - auto matcher() const - { - return match::name("slice")( - match::args(match::name("dot", "quant_dot")(match::used_once()).bind("dot_ins"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto slice_ins = r.result; - auto dot_ins = r.instructions["dot_ins"]; - auto slice_op = slice_ins->normalized_operator().to_value(); - auto axes = slice_op["axes"].to_vector(); - auto starts = slice_op["starts"].to_vector(); - auto ends = slice_op["ends"].to_vector(); - assert(starts.size() == ends.size() and starts.size() == axes.size()); - auto has_neg_vals = [](auto vec) { - return std::any_of(vec.begin(), vec.end(), [](auto i) { return i < 0; }); - }; - if(has_neg_vals(starts) or has_neg_vals(ends) or has_neg_vals(axes)) - { - MIGRAPHX_THROW("FIND_DOT_SLICE: slice is not normalized."); - } - auto dot_inputs = dot_ins->inputs(); - auto num_batch_dims = dot_ins->get_shape().lens().size() - 2; - std::vector slice_axes_1, starts_1, ends_1; // NOLINT - std::vector slice_axes_2, starts_2, ends_2; // NOLINT - for(auto i : range(axes.size())) - { - if(axes[i] < num_batch_dims) - { - slice_axes_1.push_back(axes[i]); - starts_1.push_back(starts[i]); - ends_1.push_back(ends[i]); - slice_axes_2.push_back(axes[i]); - starts_2.push_back(starts[i]); - ends_2.push_back(ends[i]); - } - else if(axes[i] == num_batch_dims) - { - slice_axes_1.push_back(axes[i]); - starts_1.push_back(starts[i]); - ends_1.push_back(ends[i]); - } - else if(axes[i] == num_batch_dims + 1) - { - slice_axes_2.push_back(axes[i]); - starts_2.push_back(starts[i]); - ends_2.push_back(ends[i]); - } - else - { - MIGRAPHX_THROW("FIND_DOT_SLICE: invalid case"); - } - } - auto slice_1 = dot_inputs.at(0); - if(not slice_axes_1.empty()) - { - slice_1 = m.insert_instruction( - slice_ins, - migraphx::make_op("slice", - {{"axes", slice_axes_1}, {"starts", starts_1}, {"ends", ends_1}}), - dot_inputs.at(0)); - } - auto slice_2 = dot_inputs.at(1); - if(not slice_axes_2.empty()) - { - slice_2 = m.insert_instruction( - slice_ins, - migraphx::make_op("slice", - {{"axes", slice_axes_2}, {"starts", starts_2}, {"ends", ends_2}}), - dot_inputs.at(1)); - } - m.replace_instruction(slice_ins, dot_ins->get_operator(), {slice_1, slice_2}); - } -}; - -struct find_dot_mul -{ - auto matcher() const - { - auto const_broadcast = match::name("broadcast", "multibroadcast")(match::is_constant()); - auto mul = match::name("mul")( - match::used_once(), - match::either_arg(0, 1)(const_broadcast.bind("d"), - match::none_of(match::is_constant()).bind("z"))); - return match::name("dot")( - match::either_arg(0, 1)(mul, match::is_constant(not_from_int4()).bind("c"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto a_ins = ins->inputs()[0]; - auto b_ins = ins->inputs()[1]; - auto d_ins = r.instructions["d"]; - auto c_ins = r.instructions["c"]; - auto z_ins = r.instructions["z"]; - - const auto& d_strides = d_ins->get_shape().strides(); - - // There should only be one stride that is not zero - if(std::count_if(d_strides.begin(), d_strides.end(), [](auto s) { return s != 0; }) > 1) - return; - - if(not d_ins->get_shape().scalar()) - { - if(d_strides.back() == 1 and not b_ins->can_eval()) - return; - if(d_strides[d_strides.size() - 2] == 1 and not a_ins->can_eval()) - return; - } - - auto broadcast_v = d_ins->get_operator().to_value(); - auto c_lens = c_ins->get_shape().lens(); - std::vector permutation(c_lens.size()); - std::iota(permutation.begin(), permutation.end(), 0); - std::swap(permutation.back(), permutation[permutation.size() - 2]); - c_lens = reorder_dims(c_lens, permutation); - broadcast_v["out_lens"] = c_lens; - auto db_ins = - m.insert_instruction(ins, make_op(d_ins->name(), broadcast_v), d_ins->inputs()); - auto db_transpose_ins = - m.insert_instruction(ins, make_op("transpose", {{"permutation", permutation}}), db_ins); - auto cd_ins = m.insert_instruction(ins, make_op("mul"), c_ins, db_transpose_ins); - - if(c_ins == b_ins) - { - a_ins = z_ins; - b_ins = cd_ins; - } - else - { - a_ins = cd_ins; - b_ins = z_ins; - } - - m.replace_instruction(ins, make_op("dot"), a_ins, b_ins); - } -}; - -// ****************************** -// a * (x + b) => a * x + a * b -// ****************************** -// When a * (x + b) is followed by another add of constant, then the -// additional add can be const folded. Also, better fusions can be applied -// when the add comes after. -struct find_mul_add -{ - auto matcher() const - { - return match::name("mul")(match::either_arg(0, 1)( - match::name("add")( - match::either_arg(0, 1)( - match::any().bind("x"), - match::any_of(conv_const_weights(), match::is_constant()).bind("b")), - match::none_of(match::args(match::is_constant(), match::is_constant())), - match::used_once()), - match::is_constant().bind("a"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto a_ins = r.instructions["a"]; - auto b_ins = r.instructions["b"]; - auto x_ins = r.instructions["x"]; - assert(x_ins != b_ins); - - auto ax_ins = m.insert_instruction(ins, make_op("mul"), a_ins, x_ins); - auto ab_ins = m.insert_instruction(ins, make_op("mul"), a_ins, b_ins); - m.replace_instruction(ins, make_op("add"), ax_ins, ab_ins); - } -}; - -struct find_dot_add -{ - auto matcher() const - { - return match::name("dot")(match::either_arg(0, 1)( - match::name("add")( - match::either_arg(0, 1)(match::any().bind("x"), - match::any_of(match::is_constant()).bind("b")), - match::none_of(match::args(match::is_constant(), match::is_constant())), - match::used_once()), - match::is_constant().bind("a"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto a_ins = r.instructions["a"]; - auto b_ins = r.instructions["b"]; - auto x_ins = r.instructions["x"]; - assert(x_ins != b_ins); - - const bool flipped = a_ins == ins->inputs().back(); - - auto insert_dot = [&](auto x, auto y) { - if(flipped) - return m.insert_instruction(ins, make_op("dot"), y, x); - else - return m.insert_instruction(ins, make_op("dot"), x, y); - }; - - auto ax_ins = insert_dot(a_ins, x_ins); - auto ab_ins = insert_dot(a_ins, b_ins); - m.replace_instruction(ins, make_op("add"), ax_ins, ab_ins); - } -}; - -struct find_conv_add -{ - auto matcher() const - { - auto add = match::name("add")( - match::either_arg(0, 1)(match::any().bind("x"), - match::any_of(match::is_constant()).bind("a")), - match::used_once()); - return match::name("convolution")(match::used_once(), - match::args(add, match::is_constant().bind("w"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto a_ins = r.instructions["a"]; - auto x_ins = r.instructions["x"]; - auto w_ins = r.instructions["w"]; - - auto conv1 = m.insert_instruction(ins, ins->get_operator(), a_ins, w_ins); - auto conv2 = m.insert_instruction(ins, ins->get_operator(), x_ins, w_ins); - - m.replace_instruction(ins, make_op("add"), conv1, conv2); - } -}; - -struct find_add_lit_broadcast -{ - auto matcher() const - { - return match::name("add")( - match::either_arg(0, 1)(op_lit_broadcast("add", "a", "x"), lit_broadcast().bind("b"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - auto a_ins = r.instructions["a"]; - auto b_ins = r.instructions["b"]; - - auto sumab = m.insert_instruction(ins, make_op("add"), a_ins, b_ins); - m.replace_instruction(ins, make_op("add"), x_ins, sumab); - } -}; - -struct find_double_add_lit_broadcast -{ - auto matcher() const - { - return match::name("add")( - match::args(op_lit_broadcast("add", "a", "x"), op_lit_broadcast("add", "b", "y"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - auto y_ins = r.instructions["y"]; - auto a_ins = r.instructions["a"]; - auto b_ins = r.instructions["b"]; - - instruction_ref sumab; - - if(a_ins->name() == "broadcast" and b_ins->name() == "broadcast") - { - if(a_ins->inputs().at(0)->get_shape() != b_ins->inputs().at(0)->get_shape()) - return; - auto op = a_ins->get_operator(); - auto presum = m.insert_instruction( - ins, make_op("add"), a_ins->inputs().at(0), b_ins->inputs().at(0)); - sumab = m.insert_instruction(ins, op, presum); - } - else - { - sumab = m.insert_instruction(ins, make_op("add"), a_ins, b_ins); - } - - auto sumxy = m.insert_instruction(ins, make_op("add"), x_ins, y_ins); - m.replace_instruction(ins, make_op("add"), sumxy, sumab); - } -}; - -/// Find elementswise operators that have all broadcast inputs. It then -/// rewrites the elementwise to do the computation on the non-broadcasted -/// axes, and then broadcast that result. -struct find_inner_broadcast -{ - auto matcher() const { return pointwise(match::all_of[match::inputs()](match::broadcast())); } - - static auto get_non_broadcast_input(instruction_ref ins) - { - if(ins->inputs().size() != 1) - return ins; - auto input = ins->inputs().front(); - if(contains(input->name(), "broadcast")) - return get_non_broadcast_input(input); - return input; - } - - static bool is_unsqueeze_needed_for_multibroadcast(const shape& input, const shape& output) - { - if(input.elements() == 1) - return false; - auto shift = output.ndim() - input.ndim(); - if(shift == 0) - return false; - if(std::equal(input.lens().begin(), - input.lens().end(), - output.lens().begin() + shift, - output.lens().end())) - { - return std::all_of(output.lens().begin(), output.lens().begin() + shift, [](auto x) { - return x == 1; - }); - } - return true; - } - // Simple case - void apply_same_broadcasts(module& m, instruction_ref ins) const - { - const auto& broadcasts = ins->inputs(); - // Scalars can have different ndim, so find the largest ndim input - auto max_broadcast = *std::max_element( - broadcasts.begin(), broadcasts.end(), by(std::less<>{}, [](instruction_ref broadcast) { - return get_non_broadcast_input(broadcast)->get_shape().ndim(); - })); - auto max_ndim = max_broadcast->get_shape().ndim(); - std::vector inputs; - std::transform(broadcasts.begin(), - broadcasts.end(), - std::back_inserter(inputs), - [&](instruction_ref broadcast) { - auto input = get_non_broadcast_input(broadcast); - auto s = input->get_shape(); - // If scalar doesnt match the other input dims then add a squeeze - if(s.elements() == 1 and s.ndim() > 1 and s.ndim() != max_ndim) - return m.insert_instruction(broadcast, make_op("squeeze"), input); - return input; - }); - auto op = insert_common_op(m, ins, ins->get_operator(), inputs); - // Find broadcast op on a non-scalar instruction if it exists - auto first = - std::find_if(broadcasts.begin(), broadcasts.end(), [&](instruction_ref broadcast) { - return not broadcast->get_shape().scalar(); - }); - if(first != broadcasts.end()) - { - m.replace_instruction(ins, (*first)->get_operator(), op); - } - else - { - m.replace_instruction(ins, broadcasts.front()->get_operator(), op); - } - } - - void apply_diff_broadcasts(module& m, instruction_ref ins) const - { - const auto& broadcasts = ins->inputs(); - auto ndim = ins->get_shape().ndim(); - // Compute the inner dimensions and axes that the computation will - // use. Also compute the axes that will be broadcasted - std::vector idims; - std::vector iaxes; - std::vector axes; - for(auto axis : range(ndim)) - { - if(std::all_of(broadcasts.begin(), broadcasts.end(), [&](instruction_ref i) { - auto s = i->get_shape(); - return s.lens()[axis] == 1 or s.strides()[axis] == 0; - })) - { - axes.push_back(axis); - } - else - { - iaxes.push_back(axis); - idims.push_back(ins->get_shape().lens()[axis]); - } - } - // If the inner axes are the same as the original operator then - // there is no reason to do this transformation. - if(iaxes.size() == ndim) - return; - std::vector inputs; - std::transform( - broadcasts.begin(), - broadcasts.end(), - std::back_inserter(inputs), - [&](instruction_ref broadcast) { - auto input = broadcast->inputs().front(); - auto s = input->get_shape(); - - // If its a single element then just return that as an input - if(s.elements() == 1) - { - if(s.lens().size() > 1) - return m.insert_instruction(broadcast, make_op("squeeze"), input); - return input; - } - - // Find how the axes are shifted from the broadcast - std::int64_t shift = ndim - s.ndim(); - if(broadcast->name() == "broadcast") - shift = broadcast->get_operator().to_value()["axis"].to(); - // Compute the squeeze axes to be used by taking the inner - // axes and shifting to what the axes will be on the - // input - std::vector sq_axes; - for(auto axis : axes) - { - auto iaxis = axis - shift; - if(iaxis < 0) - continue; - if(iaxis >= s.ndim()) - continue; - sq_axes.push_back(iaxis); - } - instruction_ref result = input; - if(not sq_axes.empty()) - result = m.insert_instruction( - broadcast, make_op("squeeze", {{"axes", sq_axes}}), result); - // If the number of dimension are still smaller than the - // number of inner axes, then we need to insert a - // broadcast to have the same dimensions for all inputs. - if(result->get_shape().ndim() < iaxes.size()) - { - // We find the first inner axis that can be mapped to the input - auto start_axis = std::find_if(iaxes.begin(), - iaxes.end(), - [&](auto x) { return x >= shift; }) - - iaxes.begin(); - result = m.insert_instruction( - broadcast, - make_op("broadcast", {{"axis", start_axis}, {"out_lens", idims}}), - result); - } - return result; - }); - auto op = insert_common_op(m, ins, ins->get_operator(), inputs); - if(iaxes.size() == 1) - { - m.replace_instruction( - ins, - make_op("broadcast", - {{"axis", iaxes.front()}, {"out_lens", ins->get_shape().lens()}}), - op); - } - else - { - auto unsqueeze = - is_unsqueeze_needed_for_multibroadcast(op->get_shape(), ins->get_shape()) - ? m.insert_instruction(ins, make_op("unsqueeze", {{"axes", axes}}), op) - : op; - m.replace_instruction( - ins, make_op("multibroadcast", {{"out_lens", ins->get_shape().lens()}}), unsqueeze); - } - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - if(ins->get_operator().name() == "layout") - return; - const auto& broadcasts = ins->inputs(); - if(broadcasts.empty()) - return; - // Skip if different data types are used - if(any_of(broadcasts, [&](auto i) { - return i->get_shape().type() != broadcasts.front()->get_shape().type(); - })) - return; - - // All inputs should have less elements - if(not all_of(broadcasts, [&](instruction_ref broadcast) { - auto input = broadcast->inputs().front(); - return input->get_shape().elements() < ins->get_shape().elements(); - })) - return; - // Find first broadcast that is not a scalar - auto first = - std::find_if(broadcasts.begin(), broadcasts.end(), [&](instruction_ref broadcast) { - return not broadcast->get_shape().scalar(); - }); - // Try to see if we can do a simple case that just applies the op to - // the inputs of the broadcasts, and then just put that same - // broadcast after the op. For this case we need each of the - // broadcasts to be the same and the inputs to have the same dimesion - // (or be scalar). - const bool same_broadcasts = - std::all_of(first, broadcasts.end(), [&](instruction_ref broadcast) { - if(broadcast->get_operator() != (*first)->get_operator()) - return false; - auto s1 = get_non_broadcast_input(broadcast)->get_shape(); - auto s2 = get_non_broadcast_input(*first)->get_shape(); - if(s1.elements() == 1) - return true; - return s1.lens() == s2.lens(); - }); - if(same_broadcasts) - { - apply_same_broadcasts(m, ins); - } - // Skip if any input to the broadcasted inputs is already broadcasted - // as the below algorithm may not be able to handle such case. - else if(std::none_of(broadcasts.begin(), broadcasts.end(), [](instruction_ref broadcast) { - return broadcast->inputs().front()->get_shape().broadcasted(); - })) - { - apply_diff_broadcasts(m, ins); - } - } -}; - -struct find_dot_broadcast -{ - auto matcher() const - { - return match::name("dot")(match::all_of[match::inputs()](match::broadcast())); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto a = ins->inputs()[0]; - auto b = ins->inputs()[1]; - if(ins->get_shape().lens().size() < 3) - return; - auto nbatch_axes = ins->get_shape().lens().size() - 2; - const auto& a_strides = a->get_shape().strides(); - const auto& b_strides = b->get_shape().strides(); - // Find leading batch axes that are broadcasted - auto p = - std::mismatch(a_strides.begin(), - a_strides.begin() + nbatch_axes, - b_strides.begin(), - b_strides.begin() + nbatch_axes, - [](auto astride, auto bstride) { return astride == 0 and bstride == 0; }); - auto naxes = p.first - a_strides.begin(); - assert(naxes <= nbatch_axes); - std::vector axes(naxes); - std::iota(axes.begin(), axes.end(), 0); - - auto insert_broadcast = [&](instruction_ref x_ins) -> instruction_ref { - auto input = x_ins->inputs()[0]; - std::vector lens(x_ins->get_shape().lens().begin() + naxes, - x_ins->get_shape().lens().end()); - - if(input->get_shape().lens() == lens) - return input; - - auto input_naxis = input->get_shape().lens().size(); - auto new_bc_naxis = lens.size(); - if(input_naxis > new_bc_naxis) - { - std::vector axes_to_sq(input_naxis - new_bc_naxis); - std::iota(axes_to_sq.begin(), axes_to_sq.end(), 0); - input = - m.insert_instruction(ins, make_op("squeeze", {{"axes", axes_to_sq}}), input); - } - - if(x_ins->name() == "multibroadcast") - { - return m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", lens}}), input); - } - else if(x_ins->name() == "broadcast") - { - auto v = x_ins->get_operator().to_value(); - auto axis = v.at("axis").to() - naxes; - return m.insert_instruction( - ins, make_op("broadcast", {{"axis", axis}, {"out_lens", lens}}), input); - } - assert(false); - return m.end(); - }; - auto a1 = insert_broadcast(a); - auto b1 = insert_broadcast(b); - auto dot = m.insert_instruction(ins, make_op("dot"), a1, b1); - auto broadcast = m.insert_instruction( - ins, make_op("multibroadcast", {{"out_lens", ins->get_shape().lens()}}), dot); - m.replace_instruction(ins, broadcast); - } -}; - -struct find_concat_op -{ - auto matcher() const - { - return match::name("concat")(match::any_of[match::inputs()]( - match::any_of(match::pointwise(), - match::name("broadcast", "multibroadcast", "unpack_int4")), - match::used_once())); - } - - template - static std::vector get_output_lens(Iterator start, Iterator last, std::size_t axis) - { - assert(start != last); - std::size_t dim = 0; - for(auto ins : range(start, last)) - { - dim += ins->get_shape().lens().at(axis); - } - auto lens = (*start)->get_shape().lens(); - lens[axis] = dim; - return lens; - } - - static bool is_valid_op(const operation& op) - { - return contains({"broadcast", "multibroadcast", "unpack_int4"}, op.name()) or - op.attributes().contains("pointwise"); - } - - static bool is_valid_concat(std::vector ins, size_t axis) - { - auto concat_lens = ins.front()->get_shape().lens(); - concat_lens.erase(concat_lens.begin() + axis); - - return std::all_of(ins.begin(), ins.end(), [&](auto i) { - auto lens = i->get_shape().lens(); - lens.erase(lens.begin() + axis); - return lens == concat_lens; - }); - } - - static bool rejected_inputs(const std::vector& inputs) - { - if(inputs.empty()) - return true; - if(inputs.size() < 3) - return false; - auto nonconst = std::count_if( - inputs.begin(), inputs.end(), [](instruction_ref ins) { return not ins->can_eval(); }); - return nonconst > 2; - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto axis = any_cast(ins->get_operator()).axis; - - auto each = [&](auto start, auto last) -> std::vector { - if(std::distance(start, last) < 2) - return {start, last}; - auto x = *start; - if(x->outputs().size() > 1 or rejected_inputs(x->inputs())) - return {start, last}; - auto op = x->get_operator(); - if(not is_valid_op(op)) - return {start, last}; - auto iaxis = axis; - // Adjust broadcast lens - if(op.name() == "broadcast") - { - auto b = any_cast(op); - if(b.axis != iaxis) - return {start, last}; - b.broadcast_lens = get_output_lens(start, last, iaxis); - op = b; - iaxis = 0; - } - else if(op.name() == "multibroadcast") - { - shape bshape = (*start)->get_shape(); - auto input = (*start)->inputs()[0]; - if(iaxis >= bshape.strides().size() or bshape.strides()[iaxis] == 0) - return {start, last}; - op.from_value({{"out_lens", get_output_lens(start, last, iaxis)}}); - auto delta = bshape.lens().size() - input->get_shape().lens().size(); - iaxis -= delta; - } - - std::vector concats; - for(std::size_t i = 0; i < x->inputs().size(); i++) - { - std::vector inputs; - std::transform(start, last, std::back_inserter(inputs), [&](auto j) { - return j->inputs().at(i); - }); - if(not is_valid_concat(inputs, iaxis)) - return {start, last}; - auto concat = - m.insert_instruction(ins, make_op("concat", {{"axis", iaxis}}), inputs); - concats.push_back(concat); - } - auto y = m.insert_instruction(ins, op, concats); - return {y}; - }; - - std::vector args; - auto update_args = [&](auto start, auto last) { - auto x = each(start, last); - args.insert(args.end(), x.begin(), x.end()); - }; - auto pred = [](auto i, auto j) { - return i->get_operator() == j->get_operator() and - i->inputs().size() == i->inputs().size() and - i->outputs().size() == i->outputs().size(); - }; - group_unique(ins->inputs().begin(), ins->inputs().end(), update_args, pred); - if(args.size() == 1) - m.replace_instruction(ins, args.front()); - else - m.replace_instruction(ins, make_op("concat", {{"axis", axis}}), args); - } -}; - -struct find_concat_conv -{ - auto matcher() const - { - return match::name("concat")( - match::all_of[match::inputs()](match::used_once(), match::name("convolution"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto axis = ins->get_operator().to_value()["axis"].to(); - if(axis != 1) - return; - if(ins->inputs().empty()) - return; - auto conv = ins->inputs().front()->get_operator(); - if(std::any_of(ins->inputs().begin(), ins->inputs().end(), [&](auto conv_ins) { - return conv_ins->get_operator() != conv; - })) - return; - std::vector inputs; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(inputs), - [](auto conv_ins) { return conv_ins->inputs()[0]; }); - if(std::any_of(inputs.begin(), inputs.end(), [&](auto input) { - return input->get_shape() != inputs.front()->get_shape(); - })) - return; - - std::vector weights; - std::transform(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(weights), - [](auto conv_ins) { return conv_ins->inputs()[1]; }); - if(std::any_of(weights.begin(), weights.end(), [&](auto w) { - return w->get_shape() != weights.front()->get_shape(); - })) - return; - - auto original_group = from_value(conv.to_value()["group"]); - auto x = m.insert_instruction(ins, make_op("concat", {{"axis", 1}}), inputs); - auto w = m.insert_instruction(ins, make_op("concat", {{"axis", 0}}), weights); - conv.from_value({{"group", original_group * inputs.size()}}); - m.replace_instruction(ins, conv, x, w); - } -}; - -void move_instructions_back(module& m, instruction_ref pos, std::vector inss) -{ - auto start = range(m.begin(), pos); - for(auto ins : iterator_for(start)) - { - auto it = std::find(inss.begin(), inss.end(), ins); - if(it != inss.end()) - inss.erase(it); - } - for(auto ins : inss) - { - if(not m.has_instruction(ins)) - continue; - move_instructions_back(m, pos, ins->inputs()); - m.move_instruction(ins, pos); - } -} - -/** Search for multiple "slice" instructions in an instruction's outputs - * which are contiguous slices of the same tensor. - */ -std::vector get_splits(instruction_ref ins) -{ - std::vector result; - std::copy_if(ins->outputs().begin(), - ins->outputs().end(), - std::back_inserter(result), - [&](auto i) { return i->name() == "slice"; }); - if(result.size() < 2) - return {}; - auto get_slice = [](auto& i) -> auto& { return any_cast(i->get_operator()); }; - auto&& axes = get_slice(result.front()).axes; - - // "slice" instructions must all have the same axes - if(std::any_of(result.begin(), result.end(), [&](auto i) { return get_slice(i).axes != axes; })) - return {}; - auto get_start = [&](auto& i) -> auto& { return get_slice(i).starts; }; - auto get_end = [&](auto& i) -> auto& { return get_slice(i).ends; }; - - // Sort the "slice" instructions in order of starts - std::sort( - result.begin(), result.end(), [&](auto x, auto y) { return get_start(x) < get_start(y); }); - if(std::any_of(get_start(result.front()).begin(), get_start(result.front()).end(), [&](auto i) { - return i != 0; - })) - return {}; - - // one slice must "start" where the last slice "end" - auto it = std::adjacent_find( - result.begin(), result.end(), [&](auto x, auto y) { return get_end(x) != get_start(y); }); - if(it != result.end()) - return {}; - for(std::size_t i = 0; i < axes.size(); i++) - { - auto axis = axes[i]; - if(ins->get_shape().lens()[axis] != get_slice(result.back()).ends[i]) - return {}; - } - return result; -} - -struct find_splits -{ - auto matcher() const - { - auto pointwise_reduction = match::any_of[match::outputs()]( - match::pointwise(match::any_of(match::nargs(1), match::nargs(2))), reduction()); - return match::any( - match::any_of[match::outputs()](match::name("slice")(pointwise_reduction))); - } - - static bool is_dependent(const module& m, instruction_ref ins1, instruction_ref ins2) - { - - std::unordered_set traversed; - return fix([&](auto self, auto ins) -> bool { - if(ins == ins2) - return true; - - if(contains(traversed, ins)) - return false; - - traversed.insert(ins); - const auto& inputs = ins->inputs(); - return std::any_of(inputs.begin(), inputs.end(), [&](auto in) { - return m.has_instruction(in) and self(in); - }); - })(ins1); - } - - static std::vector> - get_split_groups(const module& m, const std::vector& splits) - { - std::vector> groups; - for(auto out : splits.front()->outputs()) - { - if(out->name() == "slice") - continue; - std::vector group; - for(auto split : splits) - { - auto it = - std::find_if(split->outputs().begin(), split->outputs().end(), [&](auto i) { - return i->get_operator() == out->get_operator(); - }); - if(it == split->outputs().end()) - break; - assert((*it)->name() != "slice"); - - // If there is a duplicate bail - // there are should be no dependency between instructions in the group - if(std::any_of(group.begin(), group.end(), [&](auto i) { - return is_dependent(m, *it, i) or is_dependent(m, i, *it); - })) - { - return {}; - } - - group.push_back(*it); - } - if(group.size() != splits.size()) - continue; - groups.push_back(group); - } - return groups; - } - - bool is_fusable(instruction_ref start, instruction_ref split_front) const - { - auto op = start->get_operator(); - if(contains(op.name(), "reduce")) - { - auto slc = any_cast(split_front->get_operator()); - auto slc_axes = slc.axes; - auto reduce_axes = start->get_operator().to_value()["axes"].to_vector(); - // axes of slice and reduce op cannot have overlap - if(std::any_of(slc_axes.begin(), slc_axes.end(), [&](auto axis) { - return (std::find(reduce_axes.begin(), reduce_axes.end(), axis) != - reduce_axes.end()); - })) - { - return false; - } - } - else if(not op.attributes().contains("pointwise")) - { - return false; - } - - return true; - } - - int get_binary_op_split_idx(std::vector group, - std::vector splits) const - { - auto first_group_inputs = group.front()->inputs(); - auto arg_it = - std::find_if(first_group_inputs.begin(), first_group_inputs.end(), [&](auto i) { - return std::find(splits.begin(), splits.end(), i) != splits.end(); - }); - auto split_idx = arg_it - first_group_inputs.begin(); - - // All splits are at the same input index - if(std::all_of(group.begin() + 1, group.end(), [&](auto i) { - auto split_idx_input = i->inputs().at(split_idx); - return std::find(splits.begin(), splits.end(), split_idx_input) != splits.end(); - })) - return split_idx; - - return -1; - } - - void align_commutative_op_args(module& m, - std::vector group, - std::vector splits, - size_t split_idx) const - { - auto group_op = group.front()->get_operator(); - assert(std::all_of( - group.begin(), group.end(), [&](auto i) { return i->get_operator() == group_op; })); - - for(auto i : group) - { - if(std::find(splits.begin(), splits.end(), i->inputs().at(split_idx)) == splits.end()) - { - auto args = i->inputs(); - assert(args.size() == 2); - std::reverse(args.begin(), args.end()); - m.replace_instruction(i, i->get_operator(), args); - } - } - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto splits = get_splits(ins); - if(splits.empty()) - return; - - for(const auto& group : get_split_groups(m, splits)) - { - auto start = group.front(); - auto split_front = splits.front(); - auto op = start->get_operator(); - if(not is_fusable(start, split_front)) - { - continue; - } - - // Make sure there is no duplicates - assert(std::none_of( - std::next(group.begin()), group.end(), [&](auto i) { return i == start; })); - - auto split_idx = 0; - instruction_ref c = m.end(); - if(start->inputs().size() == 1) - { - c = m.insert_instruction(std::next(ins), op, ins); - } - else if(start->inputs().size() == 2) - { - assert(not std::none_of(start->inputs().begin(), start->inputs().end(), [](auto i) { - return i->name() == "slice"; - }) and "one argument must be a split"); - - split_idx = get_binary_op_split_idx(group, splits); - assert(split_idx < 2); - size_t data_idx; - if(split_idx < 0 and op.attributes().contains("commutative")) - { - split_idx = 0; - data_idx = 1; - align_commutative_op_args(m, group, splits, split_idx); - } - else if(split_idx < 0) - { - return; - } - else - { - data_idx = split_idx == 0 ? 1 : 0; - } - - std::vector data_args; - std::transform(group.begin(), - group.end(), - std::back_inserter(data_args), - [&](auto i) { return i->inputs()[data_idx]; }); - - // Data arguments must be a constant - if(std::any_of(data_args.begin(), data_args.end(), [](auto i) { - return not i->can_eval(); - })) - return; - - move_instructions_back(m, ins, data_args); - - auto slice_op = any_cast(splits.front()->get_operator()); - assert(not slice_op.axes.empty()); - if(slice_op.axes.size() > 1) - return; - auto concat_axis = slice_op.axes.front(); - // TODO: Check if axises match - auto concat = m.insert_instruction( - ins, make_op("concat", {{"axis", concat_axis}}), data_args); - - std::vector args; - args.resize(2); - args[split_idx] = ins; - args[data_idx] = concat; - c = m.insert_instruction(std::next(ins), op, args); - } - if(c != m.end()) - { - for(auto i : group) - { - auto split = i->inputs()[split_idx]; - assert(split->name() == "slice"); - - m.replace_instruction(i, split->get_operator(), c); - } - } - } - } -}; - -/** - * Matcher for a sequence of "slice" operations whose outputs are put back - * together by a "concat". - */ -struct find_split_concat -{ - auto matcher() const - { - auto concat = match::all_of[match::outputs()](match::name("concat")); - return match::any(match::any_of[match::outputs()](match::name("slice")(concat))); - } - - void apply(module& m, const match::matcher_result& r) const - { - // Verifies that the slices meet several conditions: they must all output to the same - // concat instruction, slice on the same (1 only) axis, and the end of one slice - // must match the start of the next. - auto ins = r.result; - - auto splits = get_splits(ins); - if(splits.empty()) - return; - // Each slice must output to only one instruction - if(std::any_of( - splits.begin(), splits.end(), [](auto i) { return i->outputs().size() != 1; })) - return; - // The single output instruction for all items in the list must be the same one - auto concat = splits.front()->outputs().front(); - if(std::any_of(splits.begin(), splits.end(), [&](auto i) { - return i->outputs().front() != concat; - })) - return; - - // The axis for the common output instruction must be the same as for the split ops - auto concat_op = any_cast(concat->get_operator()); - auto split_op = any_cast(splits.front()->get_operator()); - if(split_op.axes.size() != 1) - return; - if(split_op.axes.front() != concat_op.axis) - return; - - // Find where the slices are in the concat instruction's inputs (concat can have - // any number of inputs) - auto args = concat->inputs(); - auto it = - std::find_if(args.begin(), args.end(), [&](auto i) { return i == splits.front(); }); - // Verify the slices were found, and the list is long enough - if(std::distance(it, args.end()) < splits.size()) - return; - // Don't do anything if the "slice" inputs to the concat op have other operations mixed in - // among them - if(std::any_of(it, it + splits.size(), [](instruction_ref x) { - return x->get_operator().name() != "slice"; - })) - return; - // Check that the slices passed to concat are in order. - if(not std::is_sorted(it, it + splits.size(), [](instruction_ref x, instruction_ref y) { - auto xop = any_cast(x->get_operator()); - auto yop = any_cast(y->get_operator()); - return std::tie(xop.starts, xop.ends) < std::tie(yop.starts, yop.ends); - })) - return; - - // Perform the substitution - *it = splits.front()->inputs().front(); - args.erase(std::next(it), it + splits.size()); - - if(args.size() == 1) - m.replace_instruction(concat, args.front()); - else - m.replace_instruction(concat, concat->get_operator(), args); - } -}; - -bool axis_equal(const std::vector& x, - const std::vector& y, - std::size_t axis) -{ - return x.size() == y.size() and x.size() > axis and - std::equal(x.begin(), x.begin() + axis, y.begin()) and - std::equal(x.begin() + axis + 1, x.end(), y.begin() + axis + 1); -} - -bool axis_shape_equal(const shape& x, const shape& y, std::size_t axis) -{ - // TODO: Check strides - return axis_equal(x.lens(), y.lens(), axis); -} - -struct find_add_convs -{ - auto matcher() const - { - return match::name("add")( - match::args(conv_const_weights().bind("a"), conv_const_weights().bind("b"))); - } - - static bool symmetrical_strides(const op::convolution& op) - { - return op.stride[0] == op.stride[1]; - } - - static std::size_t compute_stride_factor(const op::convolution& x, const op::convolution& y) - { - if(not symmetrical_strides(x)) - return 0; - if(not symmetrical_strides(y)) - return 0; - if((x.stride[0] % y.stride[0]) != 0) - return 0; - return x.stride[0] / y.stride[0]; - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto a_conv = r.instructions["a"]; - auto a_input = a_conv->inputs().at(0); - auto a_weights = a_conv->inputs().at(1); - auto b_conv = r.instructions["b"]; - auto b_input = b_conv->inputs().at(0); - auto b_weights = b_conv->inputs().at(1); - - if(not axis_shape_equal(a_weights->get_shape(), b_weights->get_shape(), 1)) - return; - - auto a_op = any_cast(a_conv->get_operator()); - auto b_op = any_cast(b_conv->get_operator()); - auto new_op = a_op; - - if(a_op != b_op) - { - if(std::tie(a_op.padding, a_op.dilation, a_op.group) == - std::tie(b_op.padding, b_op.dilation, b_op.group) and - a_weights->get_shape().lens()[2] == 1 and a_weights->get_shape().lens()[3] == 1) - { - if(a_op.stride < b_op.stride) - { - auto n = compute_stride_factor(b_op, a_op); - if(n == 0) - return; - new_op = a_op; - b_input = m.insert_instruction( - ins, make_op("step", {{"axes", {2, 3}}, {"steps", {n, n}}}), b_input); - } - else if(b_op.stride < a_op.stride) - { - auto n = compute_stride_factor(a_op, b_op); - if(n == 0) - return; - new_op = b_op; - a_input = m.insert_instruction( - ins, make_op("step", {{"axes", {2, 3}}, {"steps", {n, n}}}), a_input); - } - else - return; - } - else - return; - } - - auto concat_input = - m.insert_instruction(ins, make_op("concat", {{"axis", 1}}), a_input, b_input); - auto concat_weights = - m.insert_instruction(ins, make_op("concat", {{"axis", 1}}), a_weights, b_weights); - m.replace_instruction(ins, new_op, concat_input, concat_weights); - } -}; - -MIGRAPHX_PRED_MATCHER(horiz_conv_dot, instruction_ref ins) -{ - auto pred = [&](auto name) { - return [=](auto i) { - return i->name() == name and i->inputs().front() == ins and - i->inputs().at(1)->can_eval(); - }; - }; - auto dots = std::count_if(ins->outputs().begin(), ins->outputs().end(), pred("dot")); - auto qdots = std::count_if(ins->outputs().begin(), ins->outputs().end(), pred("quant_dot")); - auto convs = std::count_if(ins->outputs().begin(), ins->outputs().end(), pred("convolution")); - return (dots >= 2 or convs >= 2 or qdots >= 2); -} - -struct find_conv_dot_horiz_fusion -{ - auto matcher() const { return horiz_conv_dot(); } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - - auto pred = [](auto i, auto j) { - if(i->get_operator() != j->get_operator()) - return false; - if(not contains({"quant_dot", "dot", "convolution"}, i->name())) - return true; - auto x = i->inputs()[1]->get_shape().lens(); - auto y = j->inputs()[1]->get_shape().lens(); - if(x.size() != y.size()) - return false; - // Check that non-axes match - int axis = 1; - if(i->name() == "dot" or i->name() == "quant_dot") - { - axis = x.size() - 1; - } - return axis_equal(x, y, axis); - }; - - auto each = [&](auto start, auto last) { - if(std::distance(start, last) < 2) - return; - auto&& name = (*start)->name(); - if(not contains({"quant_dot", "dot", "convolution"}, name)) - return; - auto op = (*start)->get_operator(); - int group = 1; - if(name == "convolution") - group = any_cast(op).group; - // Skip group convolution - if(group != 1) - return; - auto input = (*start)->inputs().front(); - std::vector args; - std::transform( - start, last, std::back_inserter(args), [&](auto x) { return x->inputs().at(1); }); - int axis = 1; - int concat_axis = 0; - if(name == "dot" or name == "quant_dot") - { - axis = int(args.front()->get_shape().lens().size() - 1); - concat_axis = axis; - } - - move_instructions_back(m, input, args); - // TODO: Check if axes match - auto concat = - m.insert_instruction(input, make_op("concat", {{"axis", concat_axis}}), args); - auto fused = m.insert_instruction(std::next(input), op, input, concat); - int64_t offset = 0; - for(auto arg : range(start, last)) - { - auto outputs = arg->outputs(); - - int64_t len = arg->get_shape().lens()[axis]; - m.replace_instruction( - arg, - make_op("slice", - {{"axes", {axis}}, {"starts", {offset}}, {"ends", {offset + len}}}), - fused); - offset += len; - } - }; - - auto outputs = ins->outputs(); - group_by(outputs.begin(), outputs.end(), each, pred); - } -}; - -struct find_div_const -{ - auto matcher() const - { - return match::name("div")(match::arg(1)(match::is_constant().bind("c"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto c_ins = r.instructions["c"]; - - if(shape::is_integral(ins->get_shape().type())) - return; - - auto recip = m.insert_instruction(std::next(c_ins), make_op("recip"), c_ins); - - auto args = ins->inputs(); - - m.replace_instruction(ins, make_op("mul"), args.front(), recip); - } -}; - -struct find_unit_ops -{ - auto matcher() const - { - auto mul_1 = match::name("mul")( - match::either_arg(0, 1)(match::has_value(1.0f), match::any().bind("x"))); - auto div_1 = - match::name("div")(match::args(match::any().bind("x"), match::has_value(1.0f))); - auto add_0 = match::name("add")( - match::either_arg(0, 1)(match::has_value(0.0f, 0, 0), match::any().bind("x"))); - auto sub_0 = - match::name("sub")(match::args(match::any().bind("x"), match::has_value(0.0f, 0, 0))); - return match::any_of(mul_1, div_1, add_0, sub_0); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto c_in = r.instructions["x"]; - - m.replace_instruction(ins, c_in); - } -}; - -struct find_neg_unit_ops -{ - auto matcher() const - { - auto mul_neg_1 = match::name("mul")( - match::either_arg(0, 1)(match::has_value(-1.0f), match::any().bind("x"))); - auto div_neg_1 = - match::name("div")(match::args(match::any().bind("x"), match::has_value(-1.0f))); - auto sub_0 = - match::name("sub")(match::args(match::has_value(0.0f, 0, 0), match::any().bind("x"))); - return match::any_of(mul_neg_1, div_neg_1, sub_0); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto c_in = r.instructions["x"]; - - auto neg = m.insert_instruction(ins, make_op("neg"), c_in); - m.replace_instruction(ins, neg); - } -}; - -struct eliminate_zero_point -{ - auto get_qlinear_ops_names() const - { - static std::unordered_set qdq_names = {"quantizelinear", "dequantizelinear"}; - return qdq_names; - } - auto matcher() const - { - return match::name(get_qlinear_ops_names())(match::arg(0)(match::any().bind("x")), - match::arg(1)(match::any().bind("scale")), - match::arg(2)(match::has_value(0.0f, 0, 0))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x = r.instructions["x"]; - auto scale = r.instructions["scale"]; - - auto op = ins->get_operator().to_value(); - if(ins->get_operator().name() == "quantizelinear") - { - op["out_type"] = to_value(ins->get_shape().type()); - } - auto qdq_ins = m.insert_instruction(ins, migraphx::make_op(ins->name(), op), {x, scale}); - m.replace_instruction(ins, qdq_ins); - } -}; - -struct find_zero_ops -{ - auto matcher() const - { - auto mul_zero = match::name("mul")( - match::either_arg(0, 1)(match::has_value(0.0f, 0, 0).bind("x"), match::any())); - auto div_zero = - match::name("div")(match::args(match::has_value(0.0f, 0, 0).bind("x"), match::any())); - return match::any_of(mul_zero, div_zero); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto zero_ins = r.instructions["x"]; - - m.replace_instruction(ins, zero_ins); - } -}; - -struct find_sub_const -{ - auto matcher() const - { - return match::name("sub")(match::arg(1)(match::is_constant().bind("c"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto c_ins = r.instructions["c"]; - - auto neg = m.insert_instruction(std::next(c_ins), make_op("neg"), c_ins); - - auto args = ins->inputs(); - - m.replace_instruction(ins, make_op("add"), args.front(), neg); - } -}; - -struct find_rsqrt -{ - auto matcher() const - { - auto bind_x = match::args(match::any().bind("x")); - return match::name("recip")(match::args(match::name("sqrt")(match::used_once(), bind_x))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - - m.replace_instruction(ins, make_op("rsqrt"), x_ins); - } -}; - -static bool same_ops(const std::vector& vec_ins) -{ - return std::all_of(vec_ins.begin(), vec_ins.end(), [&](auto i) { - return i->get_operator() == vec_ins.front()->get_operator(); - }); -} - -struct find_split_reshape -{ - auto matcher() const - { - auto slice_bind_slice = match::arg(0)(match::name("slice").bind("slice")); - return match::name("reshape")(match::arg(0)(match::name("contiguous")(slice_bind_slice))) - .bind("reshape"); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto slc = r.instructions["slice"]; - auto rsp = r.instructions["reshape"]; - auto input = slc->inputs().front(); - - // Only apply simplification when slices are on a single axis - auto axes = any_cast(slc->get_operator()).axes; - if(axes.size() > 1) - { - return; - } - - auto split_outputs = get_splits(input); - if(split_outputs.empty()) - { - return; - } - - // Find all the reshapes (similar to rsp) that can be simplified - std::vector conts; - std::vector vec_rsp; - - // Iterate through slice and contiguous outputs to allow simplifications when - // slice is followed by multiple reshapes - for(auto& i : split_outputs) - { - std::copy_if(i->outputs().begin(), - i->outputs().end(), - std::back_inserter(conts), - [](auto j) { return j->name() == "contiguous"; }); - } - - for(auto& i : conts) - { - std::copy_if(i->outputs().begin(), - i->outputs().end(), - std::back_inserter(vec_rsp), - [&](auto j) { return j->get_operator() == rsp->get_operator(); }); - } - - // No simplification needed if there is only one slice -> cont -> reshape - if(vec_rsp.size() <= 1) - { - return; - } - - // ensure reshape happens after the axis dimension - auto axis = axes[0]; - auto slc_lens = slc->get_shape().lens(); - auto slc_dim_size = std::accumulate( - slc_lens.begin() + axis, slc_lens.end(), 1, std::multiplies()); - auto input_lens = input->get_shape().lens(); - auto input_size = input->get_shape().elements(); - auto slc_axis_len = input_lens[axis]; - - // search the reshape output (standard shape) to decide which axis are - // in its output corresponding to the slc_dim_size - auto rsp_lens = rsp->get_shape().lens(); - auto rsp_strides = rsp->get_shape().strides(); - rsp_strides.insert(rsp_strides.begin(), rsp_strides[0] * rsp_lens[0]); - - auto ait = std::find(rsp_strides.begin(), rsp_strides.end(), slc_dim_size); - int rsp_axis = -1; - if(ait == rsp_strides.end()) - { - return; - } - else if(ait == rsp_strides.end() - 1) - { - // edge case - // slice_dim == 1, in that case it could match with last stride of 1. - // it should accumulate lengths from last dim in that case. discount 1 to avoid going - // out of bounds. - assert(slc_dim_size == 1); - rsp_axis = std::distance(rsp_strides.begin(), ait) - 1; - } - else - { - rsp_axis = std::distance(rsp_strides.begin(), ait); - } - - // Calculate reshape output shape - // Need to find a reshape such that data represented by instructions in vec_rsp can be - // written as slices of this new reshape. This is done by holding all the dims constant in - // rsp_lens to compute the required dim for rsp_axis (axis that will be sliced) - - // ex 1: Input Shape: {2, 12, 4}, Slice Axis: 1, Slices are: (0:4), (4:8), (8:12), - // Reshape Outputs: {2, 2, 2, 4}, {2, 2, 2, 4}, {2, 2, 2, 4} - // rsp_axis = 1, rsp_out_lens (initial) = {2, 1, 2, 4}, rsp_fixed_size = 2*1*2*4 = 16 - // rsp_axis_len = 2*12*4 / 16 = 6 - // rsp_out_lens (final) = {2, 6, 2, 4} - - // ex 2: Input Shape: {2, 12, 4}, Slice Axis: 1, Slices are: (0:4), (4:8), (8:12), - // Reshape Outputs: {2, 16}, {2, 16}, {2, 16} - // rsp_axis = 1, rsp_out_lens (initial) = {2, 1}, rsp_fixed_size = 2*1 = 2 - // rsp_axis_len = 2*12*4 / 2 = 48 - // rsp_out_lens (final) = {2, 48} - - std::vector rsp_out_lens(rsp_lens.begin(), rsp_lens.end()); - rsp_out_lens[rsp_axis] = 1; - auto rsp_fixed_size = std::accumulate( - rsp_out_lens.begin(), rsp_out_lens.end(), 1, std::multiplies()); - - // cannot create a valid reshape for simplification - if(input_size % rsp_fixed_size != 0) - { - return; - } - auto rsp_axis_len = input_size / rsp_fixed_size; - rsp_out_lens[rsp_axis] = rsp_axis_len; - - // Calculate new slice start and end indices. Indices are scaled using the new reshape axis - // and the original slice axis. See examples: - - // ex 1: Input Shape: {2, 12, 4}, Slice Axis: 1, Slices are: (0:4), (4:8), (8:12), - // Reshape Outputs: {2, 2, 2, 4}, {2, 2, 2, 4}, {2, 2, 2, 4} - // slc_axis_len = 12, rsp_axis_len = 6 - // New Starts: {0*6/12, 4*6/12, 8*6/12} = {0, 2, 4} - // New Ends: {4*6/12, 8*6/12, 12*6/12} = {2, 4, 6} - - // ex 2: Input Shape: {2, 12, 4}, Slice Axis: 1, Slices are: (0:4), (4:8), (8:12), - // Reshape Outputs: {2, 16}, {2, 16}, {2, 16} - // slc_axis_len = 12, rsp_axis_len = 48 - // New Starts: {0*48/12, 4*48/12, 8*48/12} = { 0, 16, 32} - // New Ends: {4*48/12, 8*48/12, 12*48/12} = {16, 32, 48} - - std::vector new_starts(vec_rsp.size()); - std::transform(vec_rsp.begin(), vec_rsp.end(), new_starts.begin(), [&](auto is) { - auto cont = is->inputs().front(); - auto og_slc = cont->inputs().front(); - return any_cast(og_slc->get_operator()).starts[0] * rsp_axis_len / - slc_axis_len; - }); - - std::vector new_ends(vec_rsp.size()); - std::transform(vec_rsp.begin(), vec_rsp.end(), new_ends.begin(), [&](auto is) { - auto cont = is->inputs().front(); - auto og_slc = cont->inputs().front(); - return any_cast(og_slc->get_operator()).ends[0] * rsp_axis_len / - slc_axis_len; - }); - - auto rsp_ins = m.insert_instruction( - std::next(input), make_op("reshape", {{"dims", rsp_out_lens}}), input); - - // replace the original reshape with slice - for(std::size_t i = 0; i < vec_rsp.size(); ++i) - { - m.replace_instruction( - vec_rsp[i], - make_op( - "slice", - {{"axes", {rsp_axis}}, {"starts", {new_starts[i]}}, {"ends", {new_ends[i]}}}), - rsp_ins); - } - } -}; - -struct find_split_transpose -{ - auto matcher() const - { - return match::name("transpose")(match::arg(0)(match::name("slice").bind("slice"))) - .bind("trans"); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto slc = r.instructions["slice"]; - auto trans = r.instructions["trans"]; - - auto input = slc->inputs().front(); - auto split_outputs = get_splits(input); - if(split_outputs.empty()) - { - return; - } - if(std::any_of(split_outputs.begin(), split_outputs.end(), [](auto i) { - return i->outputs().size() != 1; - })) - return; - - std::vector vec_trans(split_outputs.size()); - std::transform(split_outputs.begin(), split_outputs.end(), vec_trans.begin(), [](auto i) { - return i->outputs().front(); - }); - - // all transpose are the same - auto perm = any_cast(trans->get_operator()).dims; - if(not same_ops(vec_trans)) - { - return; - } - - // insert an transpose instruction - auto tr = m.insert_instruction( - std::next(input), make_op("transpose", {{"permutation", perm}}), input); - - // compute the axis in the slice - auto axis = any_cast(slc->get_operator()).axes.front(); - auto it = std::find(perm.begin(), perm.end(), axis); - assert(it != perm.end()); - int64_t axis_new = std::distance(perm.begin(), it); - - for(auto in : split_outputs) - { - auto oper = any_cast(in->get_operator()); - auto starts = oper.starts; - auto ends = oper.ends; - auto tr_orig = in->outputs().front(); - m.replace_instruction( - tr_orig, - make_op("slice", {{"axes", {axis_new}}, {"starts", starts}, {"ends", ends}}), - tr); - } - } -}; - -void simplify_algebra::apply(module& m) const -{ - // Run simplifications multiple times - m.repeat_while_changes(8, [&] { - match::find_matches(m, - find_inner_broadcast{}, - find_dot_broadcast{}, - find_double_add_lit_broadcast{}, - find_add_lit_broadcast{}, - find_add_convs{}, - find_conv_dot_horiz_fusion{}, - find_mul_conv{}, - find_mul_slice_conv{}, - find_mul_dot{}, - find_dot_slice{}, - find_dot_mul{}, - find_mul_add{}, - find_unit_ops{}, - find_neg_unit_ops{}, - eliminate_zero_point{}, - find_zero_ops{}, - find_dot_add{}, - find_conv_add{}, - find_div_const{}, - find_sub_const{}, - find_rsqrt{}, - find_concat_conv{}, - find_concat_op{}, - find_split_concat{}, - find_splits{}, - find_split_reshape{}, - find_split_transpose{}); - dead_code_elimination{}.apply(m); - }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/simplify_dyn_ops.cpp b/docker/rocm/migraphx/simplify_dyn_ops.cpp deleted file mode 100644 index 83b571a77..000000000 --- a/docker/rocm/migraphx/simplify_dyn_ops.cpp +++ /dev/null @@ -1,722 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -/** - * Convert broadcast_with_dims operators with a static input tensor and a constant `dims` input - * into multibroadcast op with a static output shape attribute. - * - */ -struct find_broadcast_with_dims_static -{ - auto matcher() const - { - return match::name("broadcast_with_dims")(match::nargs(2), - match::arg(0)(match::static_shape()), - match::arg(1)(match::is_constant())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto inputs = ins->inputs(); - - // read the values of arg(1) to create input to multibroadcast - std::vector sizes_vec; - inputs.at(1)->eval().visit( - [&](auto output) { sizes_vec.assign(output.begin(), output.end()); }); - - m.replace_instruction( - ins, make_op("multibroadcast", {{"out_lens", sizes_vec}}), inputs.at(0)); - } -}; - -/** - * Convert a Resize op. with Nearest mode to an implementation using Gather op. - * From: resize[scales={...}/sizes={...},](static, constant) - * To: - * 0 = literal{ ... } computed_indices - * ... - * 2 = reshape[dims={45}](X) 1-dimensional - * 3 = gather[axis=0](2,0) - * - * At the time of writing, this conversion is required for GPU targets because there - * is not direct a GPU implementation of the Resize operation. - * This matcher depends on a split_single_dyn_dim pass being run before it, which - * will convert any dynamic-batch input to static inputs and make this conversion possible. - * - * At time of writing, Resize allows either 1 or 2 inputs - * but the 1-input case is never created by Onnx parsing. - */ -struct find_resize_static -{ - - auto matcher() const - { - return match::name("resize")(match::nargs(2), - match::arg(0)(match::static_shape()), - match::arg(1)(match::is_constant())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto inputs = ins->inputs(); - auto resize_op = any_cast(ins->get_operator()); - - auto in_lens = inputs.at(0)->get_shape().lens(); - std::vector sizes_vec(inputs.at(0)->get_shape().ndim()); - std::vector scales_vec(inputs.at(0)->get_shape().ndim()); - // populate both scales and sizes for the benefit of the algorithm. - inputs.at(1)->eval().visit([&](auto input) { - using type = typename decltype(input)::value_type; - if constexpr(std::is_integral{}) - { - // read output sizes and use them to compute scales - sizes_vec.assign(input.begin(), input.end()); - std::transform( - input.begin(), - input.end(), - in_lens.begin(), - scales_vec.begin(), - [](auto sz, size_t in_len) { return static_cast(sz) / in_len; }); - } - else - { - // read scales and use them to compute output sizes - scales_vec.assign(input.begin(), input.end()); - std::transform( - input.begin(), - input.end(), - in_lens.begin(), - sizes_vec.begin(), - [](auto sz, size_t in_len) { return static_cast(sz * in_len); }); - } - }); - - auto in_s = inputs.at(0)->get_shape(); - shape out_s{in_s.type(), sizes_vec}; - - std::vector ind(out_s.elements()); - - // map out_idx to in_idx - auto nearest_op = op::resize::get_nearest_op(resize_op.nearest_mode); - auto idx_op = op::resize::get_original_idx_op(resize_op.coordinate_transformation_mode); - - shape_for_each(out_s, [&](const auto& out_idx_v, size_t out_idx) { - std::vector in_idx(out_idx_v.size()); - for(auto ii = 0; ii < in_lens.size(); ++ii) - { - auto idx_val = idx_op(in_lens[ii], sizes_vec[ii], out_idx_v[ii], scales_vec[ii]); - in_idx[ii] = nearest_op(in_lens[ii], idx_val); - } - - ind[out_idx] = static_cast(in_s.index(in_idx)); - }); - - // reshape input to one-dimension - std::vector rsp_lens = {static_cast(in_s.elements())}; - auto reshape_op = make_op("reshape", {{"dims", rsp_lens}}); - auto rsp = m.insert_instruction(ins, reshape_op, ins->inputs().at(0)); - - // Add our computed indices as a literal. - // ins_ind is a multi dimensional index that will restore original rank - shape ind_s{shape::int32_type, sizes_vec}; - auto ins_ind = m.add_literal(literal(ind_s, ind)); - m.replace_instruction(ins, make_op("gather", {{"axis", 0}}), rsp, ins_ind); - } -}; - -/** - * Convert 2 input static shape broadcast/multibroadcast into 1 input version. - * Some compiler passes (ex. simplify_algebra) only support the 1 input versions - * of the broadcasting operators. - * From: - * broadcast_op(argument_with_static_shape, argument_with_static_shape) - * To: - * broadcast_op(argument_with_static_shape); broadcast_op.out_lens = constant_output_dims - */ -struct find_static_2in_broadcasts -{ - auto matcher() const - { - return match::broadcast(match::nargs(2), - match::arg(0)(match::static_shape()), - match::arg(1)(match::static_shape())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto out_lens = ins->get_shape().lens(); - auto broadcast_op = ins->get_operator(); - if(broadcast_op.name() == "broadcast") - { - broadcast_op.from_value({{"out_lens", out_lens}}); - } - else - { - broadcast_op.from_value({{"out_lens", out_lens}, {"out_dyn_dims", {}}}); - } - m.replace_instruction(ins, broadcast_op, ins->inputs().at(0)); - } -}; - -/** - * Simplify slice with 2 inputs to the 1 input version if inputs[1] is constant. - * From: - * slice(data, constant_input); two attributes set - * To: - * slice(data); slice.starts, slice.ends. slice.axes set - */ -struct find_const_2in_slice -{ - auto matcher() const - { - return match::name("slice")(match::nargs(2), match::arg(1)(match::is_constant())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto inputs = ins->inputs(); - auto slice_op = any_cast(ins->get_operator()); - auto set_attrs = slice_op.get_set_attributes(); - std::vector starts_vec; - std::vector ends_vec; - std::vector axes_vec; - if(set_attrs == op::slice::ends_axes) - { - // slice(data, starts) - inputs.at(1)->eval().visit( - [&](auto output) { starts_vec.assign(output.begin(), output.end()); }); - ends_vec = slice_op.ends; - axes_vec = slice_op.axes; - } - else if(set_attrs == op::slice::starts_axes) - { - // slice(data, ends) - inputs.at(1)->eval().visit( - [&](auto output) { ends_vec.assign(output.begin(), output.end()); }); - starts_vec = slice_op.starts; - axes_vec = slice_op.axes; - } - else - { - // slice(data, axes) - inputs.at(1)->eval().visit( - [&](auto output) { axes_vec.assign(output.begin(), output.end()); }); - starts_vec = slice_op.starts; - ends_vec = slice_op.ends; - } - m.replace_instruction( - ins, - make_op("slice", {{"starts", starts_vec}, {"ends", ends_vec}, {"axes", axes_vec}}), - inputs.at(0)); - } -}; - -/** - * Simplify slice with 3 inputs to the 1 input version if inputs[1:2] are constant. - * From: - * slice(data, constant_input1, constant_input2); one attribute set - * To: - * slice(data); slice.starts, slice.ends. slice.axes set - */ -struct find_const_3in_slice -{ - auto matcher() const - { - return match::name("slice")(match::nargs(3), - match::arg(1)(match::is_constant()), - match::arg(2)(match::is_constant())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto inputs = ins->inputs(); - auto slice_op = any_cast(ins->get_operator()); - auto set_attrs = slice_op.get_set_attributes(); - std::vector starts_vec; - std::vector ends_vec; - std::vector axes_vec; - if(set_attrs == op::slice::axes_only) - { - // slice(data, starts, ends) - inputs.at(1)->eval().visit( - [&](auto output) { starts_vec.assign(output.begin(), output.end()); }); - inputs.at(2)->eval().visit( - [&](auto output) { ends_vec.assign(output.begin(), output.end()); }); - axes_vec = slice_op.axes; - } - else if(set_attrs == op::slice::ends_only) - { - // slice(data, starts, axes) - inputs.at(1)->eval().visit( - [&](auto output) { starts_vec.assign(output.begin(), output.end()); }); - inputs.at(2)->eval().visit( - [&](auto output) { axes_vec.assign(output.begin(), output.end()); }); - ends_vec = slice_op.ends; - } - else - { - // slice(data, ends, axes) - inputs.at(1)->eval().visit( - [&](auto output) { ends_vec.assign(output.begin(), output.end()); }); - inputs.at(2)->eval().visit( - [&](auto output) { axes_vec.assign(output.begin(), output.end()); }); - starts_vec = slice_op.starts; - } - m.replace_instruction( - ins, - make_op("slice", {{"starts", starts_vec}, {"ends", ends_vec}, {"axes", axes_vec}}), - inputs.at(0)); - } -}; - -/** - * Simplify slice with 4 inputs to the 1 input version if inputs[1:3] are constant. - * From: - * slice(data, constant_starts, constant_ends, constant_axes) - * To: - * slice(data); slice.starts, slice.ends. slice.axes set - */ -struct find_const_4in_slice -{ - auto matcher() const - { - return match::name("slice")(match::nargs(4), - match::arg(1)(match::is_constant()), - match::arg(2)(match::is_constant()), - match::arg(3)(match::is_constant())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto inputs = ins->inputs(); - argument starts_arg = inputs.at(1)->eval(false); - argument ends_arg = inputs.at(2)->eval(false); - argument axes_arg = inputs.at(3)->eval(false); - if(not starts_arg.empty() and not ends_arg.empty() and not axes_arg.empty()) - { - std::vector starts_vec; - std::vector ends_vec; - std::vector axes_vec; - starts_arg.visit([&](auto output) { starts_vec.assign(output.begin(), output.end()); }); - ends_arg.visit([&](auto output) { ends_vec.assign(output.begin(), output.end()); }); - axes_arg.visit([&](auto output) { axes_vec.assign(output.begin(), output.end()); }); - m.replace_instruction( - ins, - make_op("slice", {{"starts", starts_vec}, {"ends", ends_vec}, {"axes", axes_vec}}), - inputs.at(0)); - } - } -}; - -/** - * Simplify dimensions_of to a literal when the input arugment has a static shape - * or the dynamic dimensions from `start` to `end` are fixed. - */ -struct find_static_dimensions_of -{ - auto matcher() const { return match::name("dimensions_of")(); } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto input = ins->inputs().at(0); - auto dimensions_of_value = ins->get_operator().to_value(); - auto start = dimensions_of_value.at("start").to(); - auto end = dimensions_of_value.at("end").to(); - if(input->get_shape().dynamic()) - { - // check if dynamic dimensions from start to end are fixed - auto dds = input->get_shape().dyn_dims(); - if(std::any_of(dds.begin() + start, dds.begin() + end, [](auto dd) { - return not dd.is_fixed(); - })) - { - return; - } - } - std::size_t output_ndim = end - start; - std::vector vec_shape(output_ndim); - migraphx::shape s(migraphx::shape::int64_type, {output_ndim}); - std::vector input_lens = input->get_shape().to_static(1).lens(); - std::transform(input_lens.begin() + start, - input_lens.begin() + end, - vec_shape.begin(), - [](auto i) { return int64_t(i); }); - migraphx::shape output_shape{migraphx::shape::int64_type, {end - start}}; - auto lit_ins = m.add_literal(migraphx::literal{output_shape, vec_shape}); - m.replace_instruction(ins, lit_ins); - } -}; - -/** - * Simplify allocate into 2 argument reshape that has constant output dimensions into a static 1 - * argument reshape. Intended to simplify what ONNX parse_reshape creates for dynamic reshapes. - * This matcher can be generalized to matching reshape(data, static_shape_output_tensor). - * From: - * x = allocate(constant_output_dims) -> reshape(data, x) - * To: - * reshape(data); reshape.dims = constant_output_dims - */ -struct find_const_alloc_reshapes -{ - auto matcher() const - { - auto const_alloc = match::arg(1)(match::name("allocate")(match::is_constant())); - return match::name("reshape")(match::nargs(2), const_alloc); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto reshape_ins = mr.result; - auto reshape_inputs = reshape_ins->inputs(); - auto alloc_ins = reshape_inputs.at(1); - argument output_dims_arg = alloc_ins->inputs().at(0)->eval(false); - std::vector output_dims_vec; - output_dims_arg.visit( - [&](auto output) { output_dims_vec.assign(output.begin(), output.end()); }); - m.replace_instruction( - reshape_ins, make_op("reshape", {{"dims", output_dims_vec}}), reshape_inputs.at(0)); - // have dead_code_elimination remove the previous allocate - } -}; - -/** - * Simplify allocate into fill operator that has constant output dimensions and constant value. - * The allocate into fill instructions is what is produced when parsing the ONNX - * ConstantOfShape operator. This replacement could be handled with propagate_constant, but - * would rather have the simplification happen earlier during compiling. - * This matcher can be generalized to matching fill(constant_value, static_shape_output_tensor). - * From: - * x = allocate(constant_ouptut_dims) -> fill(constant_value, x) - * To: - * literal - */ -struct find_const_alloc_fill -{ - auto matcher() const - { - auto const_alloc = match::arg(1)(match::name("allocate")(match::is_constant())); - return match::name("fill")(match::arg(0)(match::is_constant()), const_alloc); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto fill_ins = mr.result; - auto fill_arg = fill_ins->eval(false); - auto l = m.add_literal(fill_arg.get_shape(), fill_arg.data()); - m.replace_instruction(fill_ins, l); - } -}; - -/** - * Simplify broadcast_for_dot instructions with two static shaped arguments - * From: - * broadcast_for_dot(static_shape_arg, static_shape_arg) - * To: - * multibroadcast(static_shape_arg); output_lens = static_broadcast_for_doted_shape - */ -struct find_static_broadcast_for_dot -{ - auto matcher() const - { - return match::name("broadcast_for_dot")(match::arg(0)(match::static_shape()), - match::arg(1)(match::static_shape())); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto broadcast_for_dot_ins = mr.result; - auto inputs = broadcast_for_dot_ins->inputs(); - auto s0 = inputs.at(0)->get_shape(); - auto s1 = inputs.at(1)->get_shape(); - auto l0_it = s0.lens().end() - 2; - std::vector l0_broadcasted_lens(s0.lens().begin(), l0_it); - auto l1_it = s1.lens().begin() + s1.ndim() - 2; - std::vector l1_broadcasted_lens(s1.lens().begin(), l1_it); - auto output_lens = compute_broadcasted_lens(l0_broadcasted_lens, l1_broadcasted_lens); - output_lens.insert(output_lens.end(), l0_it, s0.lens().end()); - m.replace_instruction(broadcast_for_dot_ins, - make_op("multibroadcast", {{"out_lens", output_lens}}), - inputs.at(0)); - } -}; - -/** - * Simplify onehot instructions with static shape `indices` input and - * a compile-time constant `depth` attribute or input. - * From: - * onehot(static_shape_arg, constant_arg, values) or - * onehot(static_shape_arg, values) - * To: - * A = literal(shape = onehot_output_shape, value = 0) - * B = unsqueeze(literal(lens = indices_lens, strides = broadcasted scalar, value = 1), - * axis=onehot_axis) C = scatter(A, unsqueeze(indices, axis=onehot_axis), B) diff = on_value - - * off_value D = mul(diff, C); return = add(D, off_value); - * - * NOTE: It might be cleaner to use some form of `fill` instead of - * (on_value - off_value) * mask + off_value when we have `fill` working - * on the GPU. - */ -struct find_static_onehot -{ - auto matcher() const - { - auto match_2_args = match::nargs(2)(match::arg(0)(match::static_shape()), - match::arg(1)(match::static_shape())); - auto match_3_args = match::nargs(3)(match::arg(0)(match::static_shape()), - match::arg(1)(match::is_constant()), - match::arg(2)(match::static_shape())); - return match::name("onehot")(match::any_of(match_2_args, match_3_args)); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto onehot_ins = mr.result; - auto onehot_inputs = onehot_ins->inputs(); - auto onehot_op = any_cast(onehot_ins->get_operator()); - auto indices_ins = onehot_inputs[0]; - shape indices_shape = indices_ins->get_shape(); - std::size_t depth_val; - migraphx::instruction_ref values_ins; - if(onehot_op.depth.has_value()) - { - assert(onehot_inputs.size() == 2); - depth_val = onehot_op.depth.value(); - values_ins = onehot_inputs[1]; - } - else - { - assert(onehot_inputs.size() == 3); - auto depth_ins = onehot_inputs[1]; - depth_ins->eval().visit([&](auto d) { depth_val = d[0]; }); - values_ins = onehot_inputs[2]; - } - shape values_shape = values_ins->get_shape(); - std::vector static_output_lens = indices_shape.lens(); - auto normalized_axis = - (onehot_op.axis < 0) ? onehot_op.axis + indices_shape.ndim() + 1 : onehot_op.axis; - static_output_lens.insert(static_output_lens.begin() + normalized_axis, depth_val); - shape output_shape{values_shape.type(), static_output_lens}; - std::vector zeros(output_shape.elements(), 0); - auto zeros_lit = m.add_literal(literal(output_shape, zeros)); - auto unsqueeze_inds = m.insert_instruction( - onehot_ins, migraphx::make_op("unsqueeze", {{"axes", {normalized_axis}}}), indices_ins); - // broadcast the one scalar to the correct shape - auto ones_lit = m.add_literal(literal(shape{values_shape.type(), {1}, {0}}, {1})); - auto mb_ones = m.insert_instruction( - onehot_ins, - migraphx::make_op("multibroadcast", {{"out_lens", unsqueeze_inds->get_shape().lens()}}), - ones_lit); - auto mask = m.insert_instruction( - onehot_ins, - make_op("scatter_none", {{"axis", normalized_axis}, {"skip_out_of_bounds", true}}), - zeros_lit, - unsqueeze_inds, - mb_ones); - auto off_val = - m.insert_instruction(onehot_ins, - make_op("slice", {{"axes", {0}}, {"starts", {0}}, {"ends", {1}}}), - values_ins); - auto on_val = - m.insert_instruction(onehot_ins, - make_op("slice", {{"axes", {0}}, {"starts", {1}}, {"ends", {2}}}), - values_ins); - auto diff_val = m.insert_instruction(onehot_ins, make_op("sub"), on_val, off_val); - auto mul_diff_mask = insert_common_op(m, onehot_ins, make_op("mul"), {diff_val, mask}); - auto mb_off_val = m.insert_instruction( - onehot_ins, make_op("multibroadcast", {{"out_lens", output_shape.lens()}}), off_val); - m.replace_instruction(onehot_ins, make_op("add"), mb_off_val, mul_diff_mask); - } -}; - -/** - * Go through `select_module` instructions and update the `output_dyn_shapes` attribute. - * Checks the submodule output shapes and determines an appropriate `output_dyn_shapes` attribute. - * This version ignores dynamic_dimension opt values. - * Intended to be run after the other simplify_dyn_ops passes. - */ -struct simplify_select_module_output_shape -{ - auto matcher() const { return match::name("select_module"); } - - void apply(module& m, const match::matcher_result& mr) const - { - auto sm_ins = mr.result; - auto sm_module_inputs = sm_ins->module_inputs(); - std::vector> all_output_shapes(sm_module_inputs.size()); - std::transform(sm_module_inputs.begin(), - sm_module_inputs.end(), - all_output_shapes.begin(), - [](auto submod) { return submod->get_output_shapes(); }); - // check that all of the submodules have the same number of outputs and all respective - // outputs have the same rank and type - auto shapes_ndim = get_shapes_ndim(all_output_shapes.front()); - auto shapes_types = get_shapes_types(all_output_shapes.front()); - if(std::any_of( - all_output_shapes.begin() + 1, all_output_shapes.end(), [&](auto out_shapes) { - bool same_types = get_shapes_types(out_shapes) == shapes_types; - bool same_ndim = get_shapes_ndim(out_shapes) == shapes_ndim; - return not same_types or not same_ndim; - })) - { - return; - } - auto num_out_shapes = shapes_ndim.size(); - std::vector dyn_shapes(num_out_shapes); - auto num_submod = sm_module_inputs.size(); - // compare respective output shapes from each submodule to get a range for the output shape - for(int i : range(num_out_shapes)) - { - std::vector shapes_at_index(num_submod); - std::transform(all_output_shapes.begin(), - all_output_shapes.end(), - shapes_at_index.begin(), - [&](auto output_shapes) { return output_shapes.at(i); }); - dyn_shapes.at(i) = dyn_shape_from_shapes(shapes_at_index); - } - auto tuple_shape = shape{dyn_shapes}; - m.replace_instruction( - sm_ins, - make_op("select_module", {{"output_dyn_shapes", to_value(tuple_shape)}}), - sm_ins->inputs(), - sm_module_inputs); - } - - std::vector get_shapes_ndim(const std::vector& shapes) const - { - std::vector ret(shapes.size()); - std::transform( - shapes.cbegin(), shapes.cend(), ret.begin(), [](auto s) { return s.ndim(); }); - return ret; - } - - std::vector get_shapes_types(const std::vector& shapes) const - { - std::vector ret(shapes.size()); - std::transform( - shapes.cbegin(), shapes.cend(), ret.begin(), [](auto s) { return s.type(); }); - return ret; - } - - /** - * Calculating an appropriate shape that encompasses all of the given vector of shapes. - * Equivalent to creating a 2D matrix of shape lengths and do a reduce over each axis. - * The shapes can be dynamic or static. - * Assuming all shapes have the same ndim. - */ - shape dyn_shape_from_shapes(std::vector shape_vec) const - { - // making 2D matrices of min_lens and max_lens - // specifically using uint64_t because we're going to put the values into a tensor_view - // later - std::vector all_min_lens; - std::vector all_max_lens; - for(const auto& s : shape_vec) - { - auto min_lens = s.min_lens(); - auto max_lens = s.max_lens(); - std::copy(min_lens.begin(), min_lens.end(), std::back_inserter(all_min_lens)); - std::copy(max_lens.begin(), max_lens.end(), std::back_inserter(all_max_lens)); - } - assert(all_min_lens.size() == shape_vec.size() * shape_vec.front().ndim()); - assert(all_max_lens.size() == shape_vec.size() * shape_vec.front().ndim()); - auto num_rows = shape_vec.size(); - auto num_cols = shape_vec.front().ndim(); - shape tensor_shape{shape::uint64_type, {num_rows, num_cols}}; - auto min_lens_matrix = make_view(tensor_shape, all_min_lens.data()); - auto max_lens_matrix = make_view(tensor_shape, all_max_lens.data()); - - std::vector mins(num_cols); - std::vector maxes(num_cols); - // rearranging data into column vectors to reduce over - // i = row, j = column - for(int j : range(num_cols)) - { - std::vector reduce_min_vals(num_rows); - std::vector reduce_max_vals(num_rows); - for(int i : range(num_rows)) - { - reduce_min_vals.at(i) = min_lens_matrix(i, j); - reduce_max_vals.at(i) = max_lens_matrix(i, j); - } - uint64_t max_int = std::numeric_limits::max(); - uint64_t min_val = - std::accumulate(reduce_min_vals.begin(), - reduce_min_vals.end(), - max_int, - [](uint64_t x, uint64_t y) { return x < y ? x : y; }); - uint64_t max_val = std::accumulate( - reduce_max_vals.begin(), reduce_max_vals.end(), 0, [](uint64_t x, uint64_t y) { - return x > y ? x : y; - }); - mins.at(j) = min_val; - maxes.at(j) = max_val; - } - // fixed output shape case - if(mins == maxes) - { - return shape{shape_vec.front().type(), mins}; - } - // dynamic output shape case - return shape{shape_vec.front().type(), mins, maxes, {}}; - } -}; - -void simplify_dyn_ops::apply(module& m) const -{ - match::find_matches(m, - find_broadcast_with_dims_static{}, - find_resize_static{}, - find_static_dimensions_of{}, - find_const_alloc_reshapes{}, - find_static_2in_broadcasts{}, - find_const_2in_slice{}, - find_const_3in_slice{}, - find_const_4in_slice{}, - find_const_alloc_fill{}, - find_static_broadcast_for_dot{}, - find_static_onehot{}); - match::find_matches(m, simplify_select_module_output_shape{}); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/simplify_qdq.cpp b/docker/rocm/migraphx/simplify_qdq.cpp deleted file mode 100644 index bd21564b6..000000000 --- a/docker/rocm/migraphx/simplify_qdq.cpp +++ /dev/null @@ -1,463 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace { - -std::unordered_set get_quantizable_op_names() -{ - static std::unordered_set s = {"convolution", "dot"}; - return s; -} - -struct match_find_quantizable_ops -{ - static bool - is_valid_qparam(instruction_ref qparam, std::vector lens, std::size_t axis) - { - return qparam->get_shape().elements() == 1 or - qparam->get_shape().elements() == lens.at(axis); - } - - static bool is_symmetric_zero_point(instruction_ref zp) - { - if(not zp->can_eval()) - return false; - - bool all_zeros = false; - zp->eval().visit([&](auto z) { - all_zeros = - std::all_of(z.begin(), z.end(), [&](auto val) { return float_equal(val, 0); }); - }); - return all_zeros; - } - - static auto - qparam_broadcast_op(instruction_ref qparam, std::vector lens, std::size_t axis) - { - if(qparam->get_shape().scalar()) - { - return migraphx::make_op("multibroadcast", {{"out_lens", lens}}); - } - else - { - return migraphx::make_op("broadcast", {{"out_lens", lens}, {"axis", axis}}); - } - } - - // Helper function to insert quantized versions of any broadcasts and transpose ops that - // occur between dequantizelinear and the quantized op - static auto propagate_quantized_ins(module& m, - const instruction_ref dqins, - const instruction_ref qop_arg, - bool is_fp16_model = false) - { - auto prev_ins = qop_arg; - std::vector ins_inbetween; - // matcher skips continguous, multi/broadcasts and transposes, collect all those - // instructions - while(prev_ins != dqins) - { - ins_inbetween.push_back(prev_ins); - prev_ins = prev_ins->inputs().front(); - } - auto qinp = dqins->inputs().front(); - for(auto ins : reverse_iterator_for(ins_inbetween)) - { - if((*ins)->name() == "convert" and is_fp16_model) - { - continue; - } - qinp = m.insert_instruction(dqins, (*ins)->get_operator(), {qinp}); - } - return qinp; - } - - auto matcher() const - { - auto dq1 = match::arg(0)( - skip_post_dq_ops(match::dequantizelinear_op("scale1", "zp1").bind("dq1"))); - auto dq2 = match::arg(1)( - skip_post_dq_ops(match::dequantizelinear_op("scale2", "zp2").bind("dq2"))); - return match::name(get_quantizable_op_names())(dq1, dq2); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto qop = r.result; - auto dq1 = r.instructions["dq1"]; - auto dq2 = r.instructions["dq2"]; - auto scale1 = r.instructions["scale1"]; - auto scale2 = r.instructions["scale2"]; - auto zp1 = r.instructions["zp1"]; - auto zp2 = r.instructions["zp2"]; - // Only INT8 or FP8 type currently supported - std::set supported_types = fp8_types{}.get(); - supported_types.insert(migraphx::shape::int8_type); - if(not contains(supported_types, dq1->inputs().front()->get_shape().type()) or - not contains(supported_types, dq2->inputs().front()->get_shape().type())) - return; - - // Propagate q1 and q2 through any broadcasts and transposes before qop - auto qop_args = qop->inputs(); - bool is_fp16_model = false; - if(dq1->get_shape().type() != qop->get_shape().type() and - qop->get_shape().type() == migraphx::shape::half_type) - { - assert(dq1->get_shape().type() == migraphx::shape::float_type); - is_fp16_model = true; - } - qop_args.at(0) = propagate_quantized_ins(m, dq1, qop_args[0], is_fp16_model); - qop_args.at(1) = propagate_quantized_ins(m, dq2, qop_args[1], is_fp16_model); - auto arg1_lens = qop_args[0]->get_shape().lens(); - auto arg2_lens = qop_args[1]->get_shape().lens(); - instruction_ref dq; - instruction_ref out_scale; - instruction_ref out_zp; - if(qop->name() == "convolution") - { - auto conv_val = qop->get_operator().to_value(); - dq = m.insert_instruction( - qop, migraphx::make_op("quant_convolution", conv_val), qop_args); - auto out_lens = dq->get_shape().lens(); - - // Ensure input and weight quantization paramaters are of a proper form - // Input is of shape [n, c, x1, ..., xn]. Only scalar quantization allowed - // Weight is of shape [k, c, y1, ... , yn]. Valid quantization axis is k - - if(not(scale1->get_shape().elements() == 1 and zp1->get_shape().elements() == 1 and - is_valid_qparam(scale2, arg2_lens, 0) and is_valid_qparam(zp2, arg2_lens, 0))) - return; - - // This implementation supports affine quantization for both input and weight - // In practice, weight is quantized symmetrically - - auto s1_bcast = - m.insert_instruction(qop, qparam_broadcast_op(scale1, out_lens, 1), scale1); - auto s2_bcast = - m.insert_instruction(qop, qparam_broadcast_op(scale2, out_lens, 1), scale2); - - out_scale = m.insert_instruction(qop, migraphx::make_op("mul"), s1_bcast, s2_bcast); - - // Compute the zero-point terms; initialize as 0 and add relevant terms - auto zero_lit = m.add_literal(literal{shape{dq->get_shape().type()}, {0}}); - out_zp = m.insert_instruction( - qop, make_op("multibroadcast", {{"out_lens", dq->get_shape().lens()}}), zero_lit); - - auto inp_zp_bc = m.insert_instruction(qop, qparam_broadcast_op(zp1, arg1_lens, 1), zp1); - auto w_zp_bc = m.insert_instruction(qop, qparam_broadcast_op(zp2, arg2_lens, 0), zp2); - - if(not is_symmetric_zero_point(zp1)) - { - auto out_zp_1 = m.insert_instruction( - qop, migraphx::make_op("quant_convolution", conv_val), inp_zp_bc, qop_args[1]); - out_zp = m.insert_instruction(qop, migraphx::make_op("add"), out_zp, out_zp_1); - } - - if(not is_symmetric_zero_point(zp2)) - { - auto out_zp_2 = m.insert_instruction( - qop, migraphx::make_op("quant_convolution", conv_val), qop_args[0], w_zp_bc); - out_zp = m.insert_instruction(qop, migraphx::make_op("add"), out_zp, out_zp_2); - } - - if(not is_symmetric_zero_point(zp1) and not is_symmetric_zero_point(zp2)) - { - auto out_zp_3 = m.insert_instruction( - qop, migraphx::make_op("quant_convolution", conv_val), inp_zp_bc, w_zp_bc); - out_zp = m.insert_instruction(qop, migraphx::make_op("sub"), out_zp, out_zp_3); - } - } - else if(qop->name() == "dot") - { - dq = m.insert_instruction(qop, migraphx::make_op("quant_dot"), qop_args); - auto out_lens = dq->get_shape().lens(); - - // For (..., M, N) x (..., N, K) dot, valid quantization axes are M for input1 and K for - // input 2 - if(not(is_valid_qparam(scale1, out_lens, out_lens.size() - 2) and - is_valid_qparam(zp1, out_lens, out_lens.size() - 2) and - is_valid_qparam(scale2, out_lens, out_lens.size() - 1) and - is_valid_qparam(zp2, out_lens, out_lens.size() - 1))) - { - return; - } - - // This implementation supports both arguments being per-axis affine quantized - // In practice, inputs are per-tensor affine and weights are per-axis symmetric - - auto s1_bcast = m.insert_instruction( - qop, qparam_broadcast_op(scale1, out_lens, out_lens.size() - 2), scale1); - auto s2_bcast = m.insert_instruction( - qop, qparam_broadcast_op(scale2, out_lens, out_lens.size() - 1), scale2); - - out_scale = m.insert_instruction(qop, migraphx::make_op("mul"), s1_bcast, s2_bcast); - - // Compute the zero-point terms; initialize as 0 and add relevant terms - auto zero_lit = m.add_literal(literal{shape{dq->get_shape().type()}, {0}}); - out_zp = m.insert_instruction( - qop, make_op("multibroadcast", {{"out_lens", dq->get_shape().lens()}}), zero_lit); - - auto zp1_bc = m.insert_instruction( - qop, qparam_broadcast_op(zp1, arg1_lens, arg1_lens.size() - 2), zp1); - auto zp2_bc = m.insert_instruction( - qop, qparam_broadcast_op(zp2, arg2_lens, arg2_lens.size() - 1), zp2); - - if(not is_symmetric_zero_point(zp1)) - { - auto out_zp_1 = - m.insert_instruction(qop, migraphx::make_op("quant_dot"), zp1_bc, qop_args[1]); - out_zp = m.insert_instruction(qop, migraphx::make_op("add"), out_zp, out_zp_1); - } - - if(not is_symmetric_zero_point(zp2)) - { - auto out_zp_2 = - m.insert_instruction(qop, migraphx::make_op("quant_dot"), qop_args[0], zp2_bc); - out_zp = m.insert_instruction(qop, migraphx::make_op("add"), out_zp, out_zp_2); - } - - if(not is_symmetric_zero_point(zp1) and not is_symmetric_zero_point(zp2)) - { - auto out_zp_3 = - m.insert_instruction(qop, migraphx::make_op("quant_dot"), zp1_bc, zp2_bc); - out_zp = m.insert_instruction(qop, migraphx::make_op("sub"), out_zp, out_zp_3); - } - } - - dq = m.insert_instruction(qop, make_op("dequantizelinear"), dq, out_scale, out_zp); - if(is_fp16_model) - { - dq = m.insert_instruction( - qop, make_op("convert", {{"target_type", migraphx::shape::half_type}}), dq); - } - m.replace_instruction(qop, dq); - } -}; - -bool compare_literals(instruction_ref ins1, instruction_ref ins2) -{ - if(ins1->name() == "broadcast" or ins1->name() == "multibroadcast") - ins1 = ins1->inputs().front(); - auto x = ins1->eval(); - if(x.empty()) - return false; - auto literal1 = ins1->get_literal(); - if(ins2->name() == "broadcast" or ins2->name() == "multibroadcast") - ins2 = ins2->inputs().front(); - auto y = ins2->eval(); - if(y.empty()) - return false; - auto literal2 = ins2->get_literal(); - - bool diff_shapes_equal_vals = false; - visit_all(ins1->get_literal(), ins2->get_literal())([&](const auto l1, const auto l2) { - diff_shapes_equal_vals = - std::all_of(l1.begin() + 1, - l1.end(), - [&](auto v) { - return ((float_equal(v, l1.front())) or - (std::isinf(static_cast(l1.front())) and - std::isinf(static_cast(v)))); - }) and - std::all_of(l2.begin(), l2.end(), [&](auto v) { - return ((float_equal(v, l1.front())) or - (std::isinf(static_cast(l1.front())) and - std::isinf(static_cast(v)))); - }); - }); - - return (x == y) or diff_shapes_equal_vals; -} - -template -bool precedes(Iterator x, Iterator y, Iterator last) -{ - auto r = range(std::next(x), last); - return any_of(iterator_for(r), [&](auto it) { return it == y; }); -} - -struct match_qlinear_reused -{ - auto matcher() const - { - return match::name("quantizelinear")( - match::used_once(), match::arg(0)(match::none_of(match::used_once()).bind("x"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - assert(ins != x_ins); - - auto dq_inputs = ins->inputs(); - dq_inputs[0] = ins; - auto outputs = x_ins->outputs(); - if(outputs.size() != 2) - return; - for(auto output : outputs) - { - if(output->name() == "quantizelinear") - continue; - if(not output->get_operator().attributes().contains("pointwise")) - continue; - if(not precedes(ins, output, m.end())) - continue; - auto dq = m.insert_instruction(std::next(ins), make_op("dequantizelinear"), dq_inputs); - instruction::replace_argument(output, x_ins, dq); - } - } -}; - -bool is_same_value(instruction_ref a, instruction_ref b) -{ - if(a == b) - return true; - return compare_literals(a, b); -} - -bool is_same_scale_zero(instruction_ref a, instruction_ref b) -{ - if(a->inputs().size() != b->inputs().size()) - return false; - if(not is_same_value(a->inputs().at(1), b->inputs().at(1))) - return false; - if(a->inputs().size() == 2) - return true; - return is_same_value(a->inputs().at(2), b->inputs().at(2)); -} - -// When an unpack instruction is inserted, its original input must be an int4/uint4. -// Therefore check for an unpack_int4 operator -- while ignoring out shape related ops. -bool is_any_input_int4(instruction_ref a) -{ - static std::set ign = {"unsqueeze", - "broadcast", - "multibroadcast", - "contiguous", - "transpose", - "reshape", - "convert"}; - return std::any_of(a->inputs().begin(), a->inputs().end(), [](auto i) { - while(ign.find(i->name()) != ign.end()) - i = i->inputs()[0]; - return i->name() == "unpack_int4"; - }); -} - -void remove_qdq_pairs(module& m) -{ - for(auto ins : iterator_for(m)) - { - auto args = ins->inputs(); - for(auto&& arg : args) - { - if(arg->name() == "dequantizelinear") - { - auto q = arg->inputs().front(); - if((q->name() == "quantizelinear") and is_same_scale_zero(arg, q)) - { - instruction::replace_argument(ins, arg, q->inputs().front()); - } - } - } - } -} - -void remove_zero_point(module& m) -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "dequantizelinear") - continue; - if(ins->inputs().size() != 3) - continue; - auto zp = ins->inputs().at(2); - if(not zp->can_eval()) - continue; - auto a = zp->eval(); - bool is_zero = false; - a.visit([&](auto t) { - is_zero = std::all_of(t.begin(), t.end(), [](auto x) { return float_equal(x, 0); }); - }); - if(not is_zero) - continue; - m.replace_instruction(ins, ins->get_operator(), ins->inputs().at(0), ins->inputs().at(1)); - } -} - -void add_int4_pack_unpack_pair(module& m) -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "dequantizelinear") - continue; - - for(auto&& inp : ins->inputs()) - { - if((inp->name() == "quantizelinear") and is_any_input_int4(inp)) - { - auto pk = m.insert_instruction(ins, make_op("pack_int4"), inp); - auto unpk = m.insert_instruction(ins, make_op("unpack_int4"), pk); - instruction::replace_argument(ins, inp, unpk); - } - } - } -} - -} // namespace - -void simplify_qdq::apply(module& m) const -{ - // first step: add pack/unpack pair between qdq for int4 weights - add_int4_pack_unpack_pair(m); - match::find_matches(m, match_find_quantizable_ops{}); - migraphx::run_passes(m, {migraphx::dead_code_elimination{}}); - remove_qdq_pairs(m); - migraphx::run_passes(m, {migraphx::dead_code_elimination{}}); - match::find_matches(m, match_qlinear_reused{}); - migraphx::run_passes(m, {migraphx::dead_code_elimination{}}); - remove_zero_point(m); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/simplify_reshapes.cpp b/docker/rocm/migraphx/simplify_reshapes.cpp deleted file mode 100644 index d74fc48c3..000000000 --- a/docker/rocm/migraphx/simplify_reshapes.cpp +++ /dev/null @@ -1,1217 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace { -const auto& reshaper_names() -{ - // clang-format off - static const std::unordered_set names = { - "flatten", - "reshape", - "contiguous", - "squeeze", - "unsqueeze" - }; - // clang-format on - return names; -} -} // namespace - -bool is_reshaper(instruction_ref ins) { return contains(reshaper_names(), ins->name()); } - -instruction_ref find_transpose_input(instruction_ref ins) -{ - if(ins->inputs().size() != 1) - return ins; - if(ins->inputs().front()->name() == "contiguous") - return find_transpose_input(ins->inputs().front()); - if(ins->inputs().front()->name() == "transpose") - return ins->inputs().front(); - return ins; -} - -auto get_transpose_dims(instruction_ref ins) -{ - return any_cast(ins->get_operator()).dims; -} - -bool is_no_transpose(const std::vector& dims) -{ - if(dims.empty()) - return true; - if(dims.front() != 0) - return false; - return std::adjacent_find( - dims.begin(), dims.end(), [](auto x, auto y) { return (y - x) != 1; }) == dims.end(); -} - -struct find_nested_shape_transforms -{ - static const auto& shape_transform_ops() - { - static const std::unordered_set names = { - "flatten", - "reshape", - "squeeze", - "unsqueeze", - "transpose", - "broadcast", - "multibroadcast", - }; - return names; - } - auto matcher() const - { - auto shape_transform = match::name(shape_transform_ops()); - auto output_not_shape_transform = - match::none_of(match::skip_output(match::name("contiguous"))(shape_transform)); - auto input_has_shape_transform = - match::args(match::skip(match::name("contiguous"))(shape_transform)); - return shape_transform(output_not_shape_transform, input_has_shape_transform); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - - std::vector ops; - auto x = ins; - while(contains(shape_transform_ops(), x->get_operator().name()) or - x->get_operator().name() == "contiguous") - { - ops.push_back(x->get_operator()); - x = x->inputs().front(); - } - if(x->get_shape().scalar()) - { - m.replace_instruction( - ins, make_op("multibroadcast", {{"out_lens", ins->get_shape().lens()}}), x); - } - else if(x->get_shape().elements() == 1 and ins->get_shape().elements() == 1) - { - // TODO: Use squeeze or unsqueeze - m.replace_instruction(ins, make_op("reshape", {{"dims", ins->get_shape().lens()}}), x); - } - else - { - std::reverse(ops.begin(), ops.end()); - auto opt_ops = optimize_shape_transforms(x->get_shape().lens(), ops); - if(ops == opt_ops) - return; - auto y = x; - for(const auto& op : opt_ops) - y = m.insert_instruction(ins, op, y); - m.replace_instruction(ins, y); - } - } -}; - -struct find_nop_reshapes -{ - auto matcher() const - { - auto reshapes = reshaper_names(); - reshapes.insert("as_shape"); - reshapes.insert("broadcast"); - reshapes.insert("concat"); - reshapes.insert("convert"); - reshapes.insert("multibroadcast"); - reshapes.insert("pad"); - reshapes.insert("slice"); - reshapes.insert("step"); - reshapes.insert("transpose"); - reshapes.insert("reduce_mean"); - reshapes.insert("reduce_max"); - reshapes.insert("reduce_min"); - reshapes.insert("reduce_sum"); - reshapes.insert("reduce_prod"); - return match::name(reshapes)(match::same_shape(match::arg(0))); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - m.replace_instruction(ins, ins->inputs().front()); - } -}; - -struct find_nested_slice -{ - auto matcher() const { return match::name("slice")(match::arg(0)(match::name("slice"))); } - - using axes_map = std::map>; - - static axes_map get_axes(instruction_ref ins) - { - axes_map result; - auto op = any_cast(ins->get_operator()); - for(std::size_t i = 0; i < op.axes.size(); i++) - { - result[op.axes[i]] = std::make_pair(op.starts[i], op.ends[i]); - } - return result; - } - - static axes_map merge(const axes_map& m1, const axes_map& m2) - { - axes_map result; - // Non overlapping - for(auto&& p : m1) - { - if(contains(m2, p.first)) - continue; - result[p.first] = p.second; - } - for(auto&& p : m2) - { - if(contains(m1, p.first)) - continue; - result[p.first] = p.second; - } - // Overlapping - for(auto&& p1 : m1) - { - if(not contains(m2, p1.first)) - continue; - auto&& v1 = p1.second; - auto&& v2 = m2.at(p1.first); - auto start = v1.first + v2.first; - auto end = start + (v2.second - v2.first); - result[p1.first] = std::make_pair(start, end); - } - return result; - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto slice = ins->inputs().front(); - auto input = slice->inputs().front(); - - auto a1 = get_axes(ins); - auto a2 = get_axes(slice); - - auto axes = merge(a2, a1); - - auto op = op::slice{}; - for(auto&& pp : axes) - { - op.axes.push_back(pp.first); - op.starts.push_back(pp.second.first); - op.ends.push_back(pp.second.second); - } - m.replace_instruction(ins, op, input); - } -}; - -/** - * Example case - * From: - * param0: lens = [3, 4], strides = [4, 1] - * param1: lens = [3, 4], strides = [4, 1] - * mb0: multibroadcast(param0, output_lens = [2, 3, 4]) - * mb1: multibroadcast(param1, output_lens = [2, 3, 4]) - * concat(mb0, mb1, axis = 2) - * - * To: - * param0: lens = [3, 4], strides = [4, 1] - * param1: lens = [3, 4], strides = [4, 1] - * con0: concat(param0, param1, axis = 1) - * multibroadcast(con0, lens = [2, 3, 4]) - */ -struct find_concat_multibroadcasts -{ - auto matcher() const - { - return match::name("concat")( - match::all_of[match::inputs()](match::name("multibroadcast", "broadcast"))); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto concat_ins = mr.result; - auto concat_op = any_cast(concat_ins->get_operator()); - auto concat_out_lens = concat_ins->get_shape().lens(); - auto concat_inputs = concat_ins->inputs(); - auto front_mb_strides = concat_inputs.front()->get_shape().strides(); - assert(concat_op.axis >= 0); - - // Only apply when concat axis is not a broadcasted dimension - if(std::any_of(concat_inputs.begin(), concat_inputs.end(), [&](auto i) { - return i->get_shape().strides()[concat_op.axis] == 0; - })) - { - return; - } - - // Skip if the broadcasts are different - auto broadcast = concat_inputs.front()->get_operator(); - auto broadcast_value = broadcast.to_value(); - if(not std::all_of(concat_inputs.begin() + 1, concat_inputs.end(), [&](instruction_ref b) { - if(b->name() != broadcast.name()) - return false; - if(broadcast.name() == "broadcast") - return b->get_operator().to_value()["axis"] == broadcast_value["axis"]; - return true; - })) - { - return; - } - - // Get the inputs of multibroadcast ops. Will be used as inputs to new concat op - std::vector inputs(concat_inputs.size()); - std::transform(concat_inputs.begin(), concat_inputs.end(), inputs.begin(), [](auto i) { - return i->inputs().front(); - }); - - // Check that the inputs into the broadcasts have the same rank - const auto& first_shape = inputs.front()->get_shape(); - if(not std::all_of(inputs.begin() + 1, inputs.end(), [&](auto input) { - return input->get_shape().ndim() == first_shape.ndim(); - })) - { - return; - } - - // Reduce axis by number of leading broadcasted dimensions - if(inputs.front()->get_shape().lens().size() < concat_out_lens.size()) - { - concat_op.axis -= - std::count(front_mb_strides.begin(), front_mb_strides.begin() + concat_op.axis, 0); - } - - // Inputs to broadcasts should have the same dimensions except for the axis to - // concatenate over - const auto& front_in_lens = inputs.front()->get_shape().lens(); - if(not std::all_of(inputs.begin() + 1, inputs.end(), [&](auto input_to_mb) { - const auto& lens = input_to_mb->get_shape().lens(); - return std::equal( - lens.begin(), lens.begin() + concat_op.axis, front_in_lens.begin()) and - std::equal(lens.begin() + concat_op.axis + 1, - lens.end(), - front_in_lens.begin() + concat_op.axis + 1); - })) - { - return; - } - - auto new_concat_ins = m.insert_instruction(concat_ins, concat_op, inputs); - broadcast.from_value({{"out_lens", concat_ins->get_shape().lens()}}); - m.replace_instruction(concat_ins, broadcast, new_concat_ins); - } -}; - -struct find_concat_slice -{ - auto matcher() const - { - return match::name("concat")(match::any_of[match::outputs()](match::name("slice"))); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto inputs = ins->inputs(); - auto outs = ins->outputs(); - std::vector slice_ins; - migraphx::transform_if( - outs.begin(), - outs.end(), - std::back_inserter(slice_ins), - [&](const auto& oins) { return oins->name() == "slice"; }, - [&](const auto& oins) { return oins; }); - int concat_axis = any_cast(ins->get_operator()).axis; - // prune slice candidates - std::vector slice_candidates; - for(const auto& sins : range(slice_ins.begin(), slice_ins.end())) - { - auto sop = any_cast(sins->get_operator()); - // slices with only one axis is allowed, because concat happens only one axis - if(sop.axes.size() != 1 or sop.axes.front() != concat_axis) - { - continue; - } - slice_candidates.push_back(sins); - } - if(slice_candidates.empty()) - { - return; - } - std::vector prefix_scan = {0}; - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(prefix_scan), [&](const auto& i) { - return prefix_scan.back() + i->get_shape().lens()[concat_axis]; - }); - for(const auto& sins : slice_candidates) - { - auto sop = any_cast(sins->get_operator()); - size_t slice_start = sop.starts.front(); - size_t slice_len = sop.ends.front() - slice_start; - auto fii = std::find_if(prefix_scan.begin(), prefix_scan.end(), [&](const auto& j) { - return j == slice_start; - }); - if(fii == prefix_scan.end()) - { - continue; - } - // slice_len == 0 - else if(fii == prefix_scan.end() - 1) - { - assert(slice_len == 0 or slice_start >= prefix_scan.back()); - continue; - } - else - { - size_t idx = std::distance(prefix_scan.begin(), fii); - if(inputs[idx]->get_shape().lens()[concat_axis] == slice_len) - { - assert((prefix_scan[idx + 1] - prefix_scan[idx]) == slice_len); - m.replace_instruction(sins, inputs[idx]); - } - } - } - } -}; - -struct find_concat_transpose -{ - auto matcher() const - { - return match::name("concat")(match::all_of[match::inputs()](match::name("transpose"))); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto trans_inputs = ins->inputs(); - auto s = trans_inputs.front()->get_shape(); - assert(s.transposed()); - auto op = any_cast(ins->get_operator()); - auto permutation = find_permutation(s); - - // permutation should be the same for all inputs - if(not std::all_of(trans_inputs.begin(), trans_inputs.end(), [&](auto in) { - return (find_permutation(in->get_shape()) == permutation); - })) - { - return; - } - - // axis could be a negative value - int64_t n_dim = s.lens().size(); - op.axis = tune_axis(n_dim, op.axis, op.name()); - - auto ipermutation = invert_permutation(permutation); - op.axis = ipermutation[op.axis]; - - std::vector inputs; - std::transform( - ins->inputs().begin(), ins->inputs().end(), std::back_inserter(inputs), [&](auto i) { - return m.insert_instruction( - ins, make_op("transpose", {{"permutation", permutation}}), i); - }); - auto concat = m.insert_instruction(ins, op, inputs); - auto t = m.insert_instruction( - ins, make_op("transpose", {{"permutation", ipermutation}}), concat); - assert(ins->get_shape().lens() == t->get_shape().lens()); - m.replace_instruction(ins, t); - } -}; - -struct find_concat_reshape -{ - auto matcher() const - { - return match::name("concat")(match::all_of[match::inputs()]( - match::name("reshape", "unsqueeze", "squeeze", "lazy_reshape"))); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto concat_shape = ins->get_shape(); - auto reshapes = ins->inputs(); - if(reshapes.empty()) - return; - auto input_shape = reshapes.front()->inputs().front()->get_shape(); - // All inputs should have the same dimensions - if(not std::all_of( - std::next(reshapes.begin()), reshapes.end(), [&](instruction_ref reshape) { - return reshape->inputs().front()->get_shape().lens() == input_shape.lens(); - })) - return; - // axis could be a negative value - auto op = any_cast(ins->get_operator()); - int64_t n_dim = reshapes.front()->get_shape().lens().size(); - auto axis = tune_axis(n_dim, op.axis, op.name()); - - auto predims = std::accumulate(concat_shape.lens().begin(), - concat_shape.lens().begin() + axis, - std::size_t{1}, - std::multiplies<>{}); - auto postdims = std::accumulate(concat_shape.lens().begin() + axis + 1, - concat_shape.lens().end(), - std::size_t{1}, - std::multiplies<>{}); - - // Find the axis on the input - std::size_t x = 1; - auto it = std::find_if(input_shape.lens().begin(), input_shape.lens().end(), [&](auto d) { - x *= d; - return x > predims; - }); - if(it == input_shape.lens().end()) - return; - op.axis = it - input_shape.lens().begin(); - auto ipredims = std::accumulate(input_shape.lens().begin(), - input_shape.lens().begin() + op.axis, - std::size_t{1}, - std::multiplies<>{}); - if(ipredims != predims) - return; - auto ipostdims = std::accumulate(input_shape.lens().begin() + op.axis + 1, - input_shape.lens().end(), - std::size_t{1}, - std::multiplies<>{}); - if(ipostdims != postdims) - return; - - std::vector inputs; - std::transform(reshapes.begin(), - reshapes.end(), - std::back_inserter(inputs), - [&](instruction_ref i) { return i->inputs().front(); }); - auto concat = m.insert_instruction(ins, op, inputs); - m.replace_instruction(ins, make_op("reshape", {{"dims", concat_shape.lens()}}), concat); - } -}; - -struct find_nested_concat -{ - auto matcher() const - { - return match::name("concat")(match::any_of[match::inputs()](match::name("concat"))); - } - - static std::size_t get_axis(instruction_ref ins) - { - auto op = any_cast(ins->get_operator()); - return op.axis; - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - auto axis = get_axis(ins); - std::vector args; - fix([&](auto self, auto&& inputs) { - for(auto&& i : inputs) - { - if(i->name() == "concat" and get_axis(i) == axis and i->outputs().size() == 1) - self(i->inputs()); - else - args.push_back(i); - } - })(ins->inputs()); - m.replace_instruction(ins, ins->get_operator(), args); - } -}; - -struct find_resize -{ - auto matcher() const - { - return match::name("gather")( - match::args(match::name("reshape").bind("data"), match::is_constant().bind("ind"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto ins_rsp = r.instructions["data"]; - auto ins_ind = r.instructions["ind"]; - - // resize input shape - if(ins_rsp->get_shape().lens().size() != 1) - { - return; - } - - // resize output shape - const auto& in_shape = ins_rsp->inputs().front()->get_shape(); - const auto& out_shape = ins->get_shape(); - // check if output shape is multiple of input shape - const auto& in_lens = in_shape.lens(); - const auto& out_lens = out_shape.lens(); - if(in_lens.size() != out_lens.size()) - { - return; - } - - // output shape must be multiple of input shape - std::vector is_multi(in_lens.size()); - std::transform( - in_lens.begin(), in_lens.end(), out_lens.begin(), is_multi.begin(), [](auto x, auto y) { - return (y % x == 0); - }); - if(not std::all_of(is_multi.begin(), is_multi.end(), [](auto b) { return b; })) - { - return; - } - - // output must be multiple of inputs - std::vector scales(in_lens.size()); - std::transform( - in_lens.begin(), in_lens.end(), out_lens.begin(), scales.begin(), [](auto x, auto y) { - return y / x; - }); - - // if ind is not constant, cannot optimize - std::vector vec_ind; - auto arg_ind = ins_ind->eval(); - if(arg_ind.empty()) - { - return; - } - arg_ind.visit([&](auto v) { vec_ind.assign(v.begin(), v.end()); }); - if(not all_of(range(out_shape.elements()), [&](auto i) { - auto out_idx = out_shape.multi(i); - auto in_idx = out_idx; - std::transform(out_idx.begin(), - out_idx.end(), - scales.begin(), - in_idx.begin(), - [&](auto io, auto scale) { return io - (io % scale); }); - return vec_ind[i] == vec_ind[out_shape.index(in_idx)]; - })) - { - return; - } - - // wrap up shapes for multibroadcast - std::vector> dim_scales; - std::transform(in_lens.begin(), - in_lens.end(), - out_lens.begin(), - std::back_inserter(dim_scales), - [](auto x, auto y) { return std::make_pair(x, y / x); }); - - std::vector in_dims; - std::vector out_dims; - for(auto& isp : dim_scales) - { - in_dims.push_back(isp.first); - out_dims.push_back(isp.first * isp.second); - if(isp.first == 1 or isp.second == 1) - { - continue; - } - - out_dims.back() = isp.first; - in_dims.push_back(1); - out_dims.push_back(isp.second); - } - - auto in_rsp = ins_rsp->inputs().front(); - auto rsp_data = m.insert_instruction( - ins_rsp, migraphx::make_op("reshape", {{"dims", in_dims}}), in_rsp); - auto mb_rsp = m.insert_instruction( - ins_rsp, migraphx::make_op("multibroadcast", {{"out_lens", out_dims}}), rsp_data); - std::vector rsp_dims(out_lens.begin(), out_lens.end()); - m.replace_instruction(ins, migraphx::make_op("reshape", {{"dims", rsp_dims}}), mb_rsp); - } -}; - -struct find_where_op -{ - auto matcher() const - { - return match::name("gather")( - match::args(match::name("reshape")(match::arg(0)(match::name("concat").bind("data"))), - match::is_constant().bind("ind"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto concat = r.instructions["data"]; - auto ins_ind = r.instructions["ind"]; - std::vector vec_ind; - auto arg_ind = ins_ind->eval(); - arg_ind.visit([&](auto v) { vec_ind.assign(v.begin(), v.end()); }); - // ind has to be the same value - auto val = vec_ind.front(); - if(not std::all_of(vec_ind.begin(), vec_ind.end(), [&](auto v) { return (v == val); })) - { - return; - } - - // concat axis must be 0 - auto op = any_cast(concat->get_operator()); - if(op.axis != 0) - { - return; - } - - // check concat inputs, it has to be 2 and have the same shape - const auto& inputs = concat->inputs(); - if(inputs.size() != 2) - { - return; - } - if(inputs.at(0)->get_shape() != inputs.at(1)->get_shape()) - { - return; - } - if(inputs.at(0)->get_shape().lens() != ins_ind->get_shape().lens()) - { - return; - } - - if(val) - { - m.replace_instruction(ins, inputs.at(0)); - } - else - { - m.replace_instruction(ins, inputs.at(1)); - } - } -}; - -struct find_reshape_cont -{ - auto matcher() const - { - auto contiguous = match::skip(match::name("contiguous"))( - match::none_of(match::standard_shape()).bind("input")); - auto reshape_contiguous = match::name("reshape")(match::args(contiguous)); - return match::pointwise( - match::nargs(2), match::either_arg(0, 1)(reshape_contiguous.bind("rsp"), match::any())); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto cont_input = r.instructions["input"]; - auto in_ins = r.instructions["rsp"]; - - auto lens = cont_input->get_shape().lens(); - std::vector dims(lens.begin(), lens.end()); - - if(in_ins->get_shape() != ins->get_shape()) - { - return; - } - - if(not std::all_of(ins->inputs().begin(), ins->inputs().end(), [](auto i) { - return i->get_shape().standard(); - })) - { - return; - } - - auto out_lens = ins->get_shape().lens(); - std::vector out_dims(out_lens.begin(), out_lens.end()); - std::vector inputs; - for(const auto& in : ins->inputs()) - { - if(in == in_ins) - { - inputs.push_back(cont_input); - } - else - { - inputs.push_back( - m.insert_instruction(ins, make_op("reshape", {{"dims", dims}}), in)); - } - } - auto out = m.insert_instruction(ins, ins->get_operator(), inputs); - m.replace_instruction(ins, make_op("reshape", {{"dims", out_dims}}), out); - } -}; - -struct find_unary_shape_transforms -{ - static const auto& shape_transforms() - { - static const std::unordered_set names = { - "flatten", - "reshape", - "squeeze", - "unsqueeze", - "transpose", - "broadcast", - "multibroadcast", - }; - return names; - } - auto matcher() const - { - auto output_not_pointwise = - match::none_of(match::skip_output(match::name("contiguous"))(match::pointwise())); - auto shape_transform = match::name(shape_transforms()); - auto input_has_shape_transform = - match::args(match::skip(match::name("contiguous"))(shape_transform)); - auto not_layout = match::none_of(match::name("layout")); - return match::pointwise( - match::used_once(), not_layout, input_has_shape_transform, output_not_pointwise); - } - - static bool is_shape_transform(instruction_ref ins) - { - return ins->inputs().size() == 1 and - (contains(shape_transforms(), ins->name()) or ins->name() == "contiguous"); - } - - static bool can_fuse_unary(instruction_ref ins) - { - return ins->name() == "@literal" or - ins->get_operator().attributes().contains("pointwise") or - contains(ins->name(), "reduce"); - } - - void apply(module& m, const match::matcher_result& mr) const - { - auto ins = mr.result; - if(ins->outputs().empty()) - return; - auto input = ins->inputs().front(); - auto output = ins->outputs().front(); - - auto insert_ops = [&](const auto& ops, instruction_ref z) { - for(const auto& op : ops) - { - z = m.insert_instruction(ins, op, z); - } - return z; - }; - - std::vector xops; - auto x = input; - while(is_shape_transform(x)) - { - xops.push_back(x->get_operator()); - x = x->inputs().front(); - } - std::reverse(xops.begin(), xops.end()); - - std::vector yops; - auto y = output; - auto last_transform = m.end(); - while(is_shape_transform(y) and y->outputs().size() == 1) - { - yops.push_back(y->get_operator()); - last_transform = y; - y = y->outputs().front(); - } - - bool move_up = can_fuse_unary(x); - bool move_down = can_fuse_unary(y); - - if(move_up and move_down) - { - if(x->name() == "@literal") - move_down = false; // NOLINT(bugprone-branch-clone) - else if(yops.empty()) - move_up = false; - else - move_down = false; - } - else if(not move_up and not move_down) - { - if(not yops.empty()) - move_up = true; - } - - if(move_up) - { - auto z = m.insert_instruction(ins, ins->get_operator(), x); - z = insert_ops(xops, z); - m.replace_instruction(ins, z); - } - else if(move_down and not yops.empty()) - { - auto z = insert_ops(yops, input); - m.replace_instruction(last_transform, ins->get_operator(), z); - } - } -}; - -struct find_slice_transpose -{ - auto matcher() const - { - auto transpose = match::output(match::name("transpose")); - return match::any(match::any_of[match::outputs()](match::name("slice")(transpose))); - } - - static std::vector find_common_perm(const std::vector& transposes) - { - std::map, int64_t> count; - for(auto t : transposes) - { - auto perm = t->get_operator().to_value()["permutation"].to_vector(); - count[perm]++; - } - return std::max_element( - count.begin(), count.end(), by(std::less<>{}, [](auto&& p) { return p.second; })) - ->first; - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - std::vector splits; - std::copy_if(ins->outputs().begin(), - ins->outputs().end(), - std::back_inserter(splits), - [&](instruction_ref out) { - return out->name() == "slice" and out->outputs().size() == 1 and - out->outputs().front()->name() == "transpose"; - }); - if(splits.size() < 2) - return; - std::vector transposes; - std::transform(splits.begin(), - splits.end(), - std::back_inserter(transposes), - [](auto split) { return split->outputs().front(); }); - auto perm = find_common_perm(transposes); - auto iperm = invert_permutation(perm); - auto pre = m.insert_instruction( - std::next(ins), make_op("transpose", {{"permutation", perm}}), ins); - for(auto i : range(transposes.size())) - { - auto split = splits[i]; - auto t = transposes[i]; - auto op = any_cast(split->get_operator()); - std::transform(op.axes.begin(), op.axes.end(), op.axes.begin(), [&](auto axis) { - return iperm[axis]; - }); - auto new_ins = m.insert_instruction(t, op, pre); - if(t->get_operator() != pre->get_operator()) - { - auto curr = t->get_operator().to_value()["permutation"].to_vector(); - new_ins = m.insert_instruction( - t, make_op("transpose", {{"permutation", reorder_dims(iperm, curr)}}), new_ins); - } - m.replace_instruction(t, new_ins); - } - } -}; - -struct find_transpose_slice -{ - auto matcher() const - { - return match::name("transpose")(match::all_of[match::outputs()](match::name("slice"))); - } - - static std::vector slice_distance(const op::slice& op) - { - assert(op.starts.size() == op.ends.size()); - std::vector result(op.starts.size()); - std::transform( - op.ends.begin(), op.ends.end(), op.starts.begin(), result.begin(), std::minus<>{}); - return result; - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto slices = ins->outputs(); - if(slices.empty()) - return; - auto slice = any_cast(slices.front()->get_operator()); - auto sdistance = slice_distance(slice); - // Check all distances and axes are the same - if(std::any_of(slices.begin(), slices.end(), [&](auto sins) { - auto s = any_cast(sins->get_operator()); - return s.axes != slice.axes or slice_distance(s) != sdistance; - })) - return; - // Check distances are divisible by lens of corresponding axes - auto mod_by_distance = [&](const auto& v, auto f) { - return std::inner_product(v.begin(), - v.end(), - sdistance.begin(), - 0, - std::plus<>{}, - [&](auto x, auto d) -> uint64_t { - if(d == 0) - return 1; - return f(x) % d; - }); - }; - if(mod_by_distance(slice.axes, [&](auto x) { return ins->get_shape().lens()[x]; }) != 0 or - mod_by_distance(slice.starts, id{}) != 0 or mod_by_distance(slice.ends, id{}) != 0) - return; - // TODO: Handle multiple axes - if(sdistance.size() != 1) - return; - auto axis = slice.axes.front(); - // Skip if axis would be packed - if(std::all_of(ins->get_shape().lens().begin(), - ins->get_shape().lens().begin() + axis, - [](auto x) { return x == 1; })) - return; - // Compute axis before transpose to use for unsqueeze - auto perm = ins->get_operator().to_value()["permutation"].to_vector(); - auto preaxis = perm[axis]; - // Make unsqueeze - std::vector steps(sdistance.size()); - std::transform( - slice.axes.begin(), - slice.axes.end(), - sdistance.begin(), - steps.begin(), - [&](const auto ax, const auto sdis) { return ins->get_shape().lens().at(ax) / sdis; }); - auto unsqueeze = m.insert_instruction( - ins, make_op("unsqueeze", {{"axes", {preaxis}}, {"steps", steps}}), ins->inputs()); - // Make transpose - std::transform(perm.begin(), perm.end(), perm.begin(), [&](auto i) { - if(i >= preaxis) - return i + 1; - return i; - }); - perm.insert(perm.begin(), preaxis); - auto transpose = - m.insert_instruction(ins, make_op("transpose", {{"permutation", perm}}), unsqueeze); - // Slice and squeeze - for(auto s : slices) - { - auto op = any_cast(s->get_operator()); - op.axes = {0}; - op.starts = {op.starts.front() / sdistance.front()}; - op.ends = {op.ends.front() / sdistance.front()}; - auto slice_ins = m.insert_instruction(ins, op, transpose); - auto squeeze = - m.insert_instruction(ins, make_op("squeeze", {{"axes", {0}}}), slice_ins); - m.replace_instruction(s, squeeze); - } - } -}; - -struct find_reshape_dot -{ - auto matcher() const - { - auto rsp = match::name("reshape").bind("rsp"); - auto other = match::skip_broadcasts(match::any().bind("other")); - return match::name("dot")(match::used_once(), match::either_arg(0, 1)(rsp, other)); - } - - // Gemm axis should not be altered by the reshape - auto is_valid_reshape(instruction_ref inp, instruction_ref rsp, size_t dot_axis) const - { - auto inp_lens = inp->get_shape().lens(); - auto rsp_lens = rsp->get_shape().lens(); - - return (inp_lens.size() >= dot_axis and - rsp_lens[rsp_lens.size() - dot_axis] == inp_lens[inp_lens.size() - dot_axis]); - } - - // Same batch dims - auto has_same_batch_dims(instruction_ref in1, instruction_ref in2) const - { - auto in1_lens = in1->get_shape().lens(); - auto in2_lens = in2->get_shape().lens(); - - return ( - in1_lens.size() == in2_lens.size() and - std::equal(in1_lens.begin(), in1_lens.end() - 2, in2_lens.begin(), in2_lens.end() - 2)); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto dot = r.result; - auto rsp = r.instructions["rsp"]; - auto other = r.instructions["other"]; - - auto rsp_lens = rsp->get_shape().lens(); - auto inp = rsp->inputs().front(); - auto inp_lens = inp->get_shape().lens(); - - // Gemm axis should not be altered by the reshape - bool flipped = rsp == dot->inputs().back(); - size_t dot_axis = (flipped) ? 2 : 1; - - if(not is_valid_reshape(inp, rsp, dot_axis)) - return; - - instruction_ref new_other; - if(other->get_operator().name() == "reshape") - { - auto other_inp = other->inputs().front(); - size_t other_dot_axis = (flipped) ? 1 : 2; - if(not is_valid_reshape(other_inp, other, other_dot_axis) or - not has_same_batch_dims(inp, other_inp)) - return; - - new_other = other_inp; - } - else - { - auto other_lens = other->get_shape().lens(); - if(other_lens.size() > 2) - return; - - std::vector new_other_lens{inp_lens.begin(), inp_lens.end() - 2}; - operation new_bc_op; - - auto bc_other = (flipped) ? dot->inputs().front() : dot->inputs().back(); - auto bc_other_lens = bc_other->get_shape().lens(); - new_other_lens.insert( - new_other_lens.end(), bc_other_lens.end() - 2, bc_other_lens.end()); - - // if the original weight is one dimensional, look at the original broadcast - // to determine the correct broadcast axis - if(other_lens.size() == 1) - { - auto bc_other_strides = bc_other->get_shape().strides(); - auto it = std::find_if(bc_other_strides.begin(), - bc_other_strides.end(), - [&](auto i) { return i != 0; }); - auto orig_bc_axis = std::distance(bc_other_strides.begin(), it); - - auto new_bc_axis = new_other_lens.size() - (bc_other_lens.size() - orig_bc_axis); - new_bc_op = - make_op("broadcast", {{"axis", new_bc_axis}, {"out_lens", new_other_lens}}); - } - else - { - new_bc_op = make_op("multibroadcast", {{"out_lens", new_other_lens}}); - } - - new_other = m.insert_instruction(dot, new_bc_op, other); - } - - instruction_ref new_dot; - if(flipped) - { - new_dot = m.insert_instruction(dot, make_op("dot"), new_other, inp); - } - else - { - new_dot = m.insert_instruction(dot, make_op("dot"), inp, new_other); - } - m.replace_instruction( - dot, make_op("reshape", {{"dims", dot->get_shape().lens()}}), new_dot); - } -}; - -// Remove transposes and converts between mul/add -> dot so simplify_algebra can perform -// const folding simplifications -struct find_mul_add_shape_op_dot -{ - auto matcher() const - { - auto shape_ops = match::name("transpose", "convert"); - auto const_mul_add = match::name("mul", "add")(match::either_arg(0, 1)( - match::is_constant().bind("const"), match::any().bind("input"))); - auto match_shape_op = shape_ops(match::args(const_mul_add.bind("pw"))); - auto skip_shape_op_outputs = match::skip_output(match::any_of(shape_ops)); - return match_shape_op(skip_shape_op_outputs(match::name("dot"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto shape_ins = r.result; - auto pw = r.instructions["pw"]; - auto constant = r.instructions["const"]; - auto input = r.instructions["input"]; - - auto shape_op = shape_ins->get_operator(); - auto pw_op = pw->get_operator(); - auto new_inp = m.insert_instruction(shape_ins, shape_op, input); - auto new_const = m.insert_instruction(shape_ins, shape_op, constant); - - m.replace_instruction(shape_ins, pw_op, new_inp, new_const); - } -}; - -struct find_flatten -{ - auto matcher() const { return match::name("flatten"); } - - void apply(module& m, const match::matcher_result& r) const - { - auto flatten = r.result; - m.replace_instruction(flatten, - make_op("reshape", {{"dims", flatten->get_shape().lens()}}), - flatten->inputs()); - } -}; - -void simplify_reshapes::apply(module& m) const -{ - m.repeat_while_changes(depth, [&] { - match::find_matches(m, - find_where_op{}, - find_resize{}, - find_nop_reshapes{}, - find_flatten{}, - find_reshape_cont{}, - find_nested_shape_transforms{}, - find_concat_slice{}, - find_concat_transpose{}, - find_concat_reshape{}, - find_concat_multibroadcasts{}, - find_nested_slice{}, - find_nested_concat{}, - find_transpose_slice{}, - find_slice_transpose{}, - find_unary_shape_transforms{}, - find_reshape_dot{}, - find_mul_add_shape_op_dot{}); - dead_code_elimination{}.apply(m); - }); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/split_reduce.cpp b/docker/rocm/migraphx/split_reduce.cpp deleted file mode 100644 index 3188b0056..000000000 --- a/docker/rocm/migraphx/split_reduce.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct split_fused_reduce -{ - std::vector axes{}; - std::string assign = "assign_none"; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.axes, "axes"), f(self.assign, "assign")); - } - - value attributes() const { return {{"prefill", 0}}; } - - shape compute_shape(const std::vector& inputs, std::vector mods) const - { - if(mods.size() != 1) - MIGRAPHX_THROW("should have one submodule."); - const auto* sm = mods.front(); - auto names = sm->get_parameter_names(); - check_shapes{inputs, *this}.has(names.size()).same_ndims(); - - auto result = - sm->compute_shapes(inputs, {.name = name(), .strict_type = true, .strict_lens = true}); - if(result.size() == 1) - return result.front(); - return shape{result}; - } - - std::string name() const { return "split_fused_reduce"; } -}; -MIGRAPHX_REGISTER_OP(split_fused_reduce); - -static bool is_reduce(const instruction& ins) { return contains(ins.name(), "reduce"); } - -namespace { -struct splitter -{ - const_module_ref rm; - - bool strictly_dominate(instruction_ref a, instruction_ref b) - { - if(not dom.has_value()) - dom = compute_dominator(*rm); - return dom->strictly_dominate(a, b); - } - - std::vector find_splits() const - { - std::vector result; - copy_if(iterator_for(*rm), std::back_inserter(result), [](auto ins) { - return is_reduce(*ins); - }); - if(result.size() > 2) - return {}; - // Only handle reduce_sum for now - // TODO: Support other reduction types - if(not std::all_of(result.begin(), result.end(), [](instruction_ref ins) { - return ins->name() == "reduce_sum"; - })) - return {}; - if(result.size() < 2) - return result; - if(reaches(result[0], result[1])) - return {}; - return result; - } - - std::vector find_alive(const std::vector& splits) - { - std::vector result; - bool stop = false; - liveness(*rm, [&](auto rins, const auto& live_set) { - if(stop) - return; - if(rins == rm->begin()) - return; - // We want to know what instructions are live after the split instruction - auto ins = instruction::get_output_alias(std::prev(rins)); - if(not contains(splits, ins)) - return; - std::copy_if(live_set.begin(), - live_set.end(), - std::back_inserter(result), - [&](instruction_ref live) { - if(live->name() == "@param") - return false; - if(contains(splits, live)) - return false; - if(splits.size() > 1 and none_of(splits, [&](instruction_ref split) { - return this->strictly_dominate(live, split); - })) - return false; - return true; - }); - stop = true; - }); - return result; - } - - std::optional dom = std::nullopt; -}; -} // namespace - -static std::string assign_op(const std::vector& splits) -{ - static std::unordered_map m = { - {"reduce_sum", "assign_add"}, - {"reduce_mean", "assign_add"}, - {"reduce_prod", "assign_mul"}, - {"reduce_max", "assign_max"}, - {"reduce_min", "assign_min"}, - }; - return m.at(splits.front()->name()); -} - -static std::vector -insert_module_inline(module& m, instruction_ref ins, const module::with_inputs& mwi) -{ - auto param_map = mwi.mod.get_ins_param_map(mwi.inputs, true); - return m.insert_instructions(ins, &mwi.mod, ¶m_map); -} - -static std::size_t get_reduce_size(const_module_ref rm) -{ - auto ins = std::find_if(rm->begin(), rm->end(), &is_reduce); - assert(ins != rm->end()); - return ins->inputs().front()->get_shape().elements() / ins->get_shape().elements(); -} - -void split_reduce::apply(module_pass_manager& mpm) const -{ - for(auto ins : iterator_for(mpm.get_module())) - { - if(ins->name() != "fused_reduce") - continue; - auto* rm = ins->module_inputs().front(); - if(get_reduce_size(rm) < split_size) - continue; - splitter s{rm}; - auto splits = s.find_splits(); - if(splits.empty()) - continue; - // Only use split reduce with float for now - // TODO: Support other data types - if(not std::all_of(splits.begin(), splits.end(), [](instruction_ref split) { - return contains({shape::float_type, shape::half_type}, split->get_shape().type()); - })) - continue; - auto v = ins->get_operator().to_value(); - auto axes = v["axes"].to_vector(); - - auto alive = s.find_alive(splits); - - std::array mods; - if(not alive.empty()) - { - auto mods3 = rm->split(ins->inputs(), alive, splits); - auto r = insert_module_inline(mpm.get_module(), ins, mods3[0]); - mods3[1].replace(alive, r); - mods3[2].replace(alive, r); - mods = {std::move(mods3[1]), std::move(mods3[2])}; - } - else - { - mods = rm->split(ins->inputs(), splits); - } - - auto* splitm = mpm.create_module(rm->name() + "_split", std::move(mods[0].mod)); - splitm->set_bypass(); - - // Insert split reduce - auto split_reduce = mpm.get_module().insert_instruction( - ins, - make_op("split_fused_reduce", {{"axes", axes}, {"assign", assign_op(splits)}}), - mods[0].inputs, - {splitm}); - - std::vector split_reduce_each; - if(splits.size() == 1) - { - split_reduce_each = {split_reduce}; - } - else - { - transform(range(splits.size()), std::back_inserter(split_reduce_each), [&](auto i) { - return mpm.get_module().insert_instruction( - ins, make_op("get_tuple_elem", {{"index", i}}), split_reduce); - }); - } - - mods[1].replace(splits, split_reduce_each); - auto replaced = insert_module_inline(mpm.get_module(), ins, mods[1]); - assert(replaced.size() == 1); - mpm.get_module().replace_instruction(ins, replaced.front()); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/split_single_dyn_dim.cpp b/docker/rocm/migraphx/split_single_dyn_dim.cpp deleted file mode 100644 index 599889672..000000000 --- a/docker/rocm/migraphx/split_single_dyn_dim.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct dynamic_dimensions_check -{ - std::string dyn_param_str; - shape::dynamic_dimension dd; -}; - -/** - * Returns value if the parameters contain non-fixed dynamic_dimensions that are the same between - * all of the dynamic shape parameters. - * In other words, each parameter can have one non-fixed dynamic_dimension `x` where `x` is the same - * between all of the parameters with a non-fixed dynamic_dimension. - * Returns the parameters and the dynamic dimension in a vector of dynamic_dimensions_check objects. - */ -optional> -has_one_unique_dyn_dim(const std::unordered_map& param_shapes) -{ - auto is_dynamic = [](const auto& p) { return p.second.dynamic(); }; - std::vector::value_type> dyn_params{}; - std::copy_if( - param_shapes.begin(), param_shapes.end(), std::back_inserter(dyn_params), is_dynamic); - if(dyn_params.empty()) - return std::nullopt; - std::vector ret{}; - // get non-fixed dynamic_dimension from all parameters - for(const auto& param : dyn_params) - { - const auto& dds = param.second.dyn_dims(); - auto num_non_fixed = std::count_if(dds.cbegin(), dds.cend(), [&](auto dd) { - if(not dd.is_fixed()) - { - ret.push_back(dynamic_dimensions_check{param.first, dd}); - return true; - } - return false; - }); - // catch more than one non-fixed dynamic_dimension - if(num_non_fixed > 1) - { - return std::nullopt; - } - } - if(ret.empty()) - { - return std::nullopt; - } - // check all the same dynamic_dimension - bool same_dd = - std::all_of(ret.begin() + 1, ret.end(), [&](auto ddc) { return ddc.dd == ret.at(0).dd; }); - if(same_dd) - { - return ret; - } - return std::nullopt; -} - -/** - * Check the parameters in std::vector object to see if any of the - * parameters outputs to a select_module operator. - */ -bool any_sm_next(const_module_ref mm, const std::vector& ddcs) -{ - for(const auto& ddc : ddcs) - { - auto p_outputs = mm->get_parameter(ddc.dyn_param_str)->outputs(); - bool is_sm_next = std::any_of(p_outputs.cbegin(), p_outputs.cend(), [](auto ins) { - return ins->name() == "select_module"; - }); - if(is_sm_next) - { - return true; - }; - } - return false; -} - -/** - * Makes all the shapes in the dynamic_dimension range. Probably won't work for `if` - * and `loop` instructions, depending on how the submodules for those - * work. Inserts select_module instruction to the top. Replaces return, bypassing other - * instructions. Skips if the dynamic parameter outputs to a select_module operator. - */ -void split_single_dyn_dim::apply(module_pass_manager& mpm) const -{ - module_ref mm = &mpm.get_module(); - auto param_names = mm->get_parameter_names(); - auto param_shapes = mm->get_parameter_shapes(); - optional> dd_check_vec = - has_one_unique_dyn_dim(param_shapes); - if(dd_check_vec.has_value() and not any_sm_next(mm, dd_check_vec.value())) - { - // all dynamic dimension objects should be the same for all parameters in dd_check_vec - auto dyn_dim = dd_check_vec->at(0).dd; - // create submodules for each dimension size - std::vector submodules; - for(size_t dim_size : migraphx::range(dyn_dim.min, dyn_dim.max + 1)) - { - auto* submod = mpm.create_module("dim_" + std::to_string(dim_size)); - // instruction map for new static shaped submodule parameters - std::unordered_map map_ins; - for(const auto& dd_check : dd_check_vec.value()) - { - // create static shape using dim_size - const auto& dyn_param = mm->get_parameter(dd_check.dyn_param_str); - auto dyn_param_shape = mm->get_parameter_shape(dd_check.dyn_param_str); - auto static_shape = dyn_param_shape.to_static(dim_size); - map_ins[dyn_param] = submod->add_parameter(dd_check.dyn_param_str, static_shape); - } - auto outputs = submod->add_instructions(mm, &map_ins); - submod->add_return({outputs}); - submodules.push_back(submod); - } - // sort parameters by name for consistency (vs. parameter order attr) - std::sort(param_names.begin(), param_names.end()); - // redirect to select_module operator and return - std::vector sm_inputs; - std::transform(param_names.cbegin(), - param_names.cend(), - std::back_inserter(sm_inputs), - [&](auto pn) { return mm->get_parameter(pn); }); - auto output_shapes = mm->get_output_shapes(); - migraphx::shape out_attr = migraphx::shape{output_shapes}; - auto sm_ins = mm->add_instruction( - migraphx::make_op("select_module", - {{"output_dyn_shapes", migraphx::to_value(out_attr)}}), - sm_inputs, - submodules); - std::vector outputs(output_shapes.size()); - for(size_t i = 0; i < output_shapes.size(); ++i) - { - outputs.at(i) = - mm->add_instruction(migraphx::make_op("get_tuple_elem", {{"index", i}}), sm_ins); - } - mm->replace_return(outputs); - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/sqlite.cpp b/docker/rocm/migraphx/sqlite.cpp deleted file mode 100644 index 823d74c3a..000000000 --- a/docker/rocm/migraphx/sqlite.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -using sqlite3_ptr = MIGRAPHX_MANAGE_PTR(sqlite3*, sqlite3_close); - -struct sqlite_impl -{ - sqlite3* get() const { return ptr.get(); } - void open(const fs::path& p, int flags) - { - sqlite3* ptr_tmp = nullptr; - int rc = sqlite3_open_v2(p.string().c_str(), &ptr_tmp, flags, nullptr); - ptr = sqlite3_ptr{ptr_tmp}; - if(rc != 0) - MIGRAPHX_THROW("error opening " + p.string() + ": " + error_message()); - } - - template - void exec(const char* sql, F f) - { - // cppcheck-suppress constParameterPointer - auto callback = [](void* obj, auto... xs) -> int { - try - { - const auto* g = static_cast(obj); - (*g)(xs...); - return 0; - } - catch(...) - { - return -1; - } - }; - int rc = sqlite3_exec(get(), sql, callback, &f, nullptr); - if(rc != 0) - MIGRAPHX_THROW(error_message()); - } - - std::string error_message() const - { - std::string msg = "sqlite3: "; - return msg + sqlite3_errmsg(get()); - } - sqlite3_ptr ptr; -}; - -sqlite sqlite::read(const fs::path& p) -{ - sqlite r; - r.impl = std::make_shared(); - r.impl->open(p, SQLITE_OPEN_READONLY); - return r; -} - -sqlite sqlite::write(const fs::path& p) -{ - sqlite r; - r.impl = std::make_shared(); - // Using '+' instead of bitwise '|' to avoid compilation warning - r.impl->open(p, SQLITE_OPEN_READWRITE + SQLITE_OPEN_CREATE); - return r; -} - -std::vector> sqlite::execute(const std::string& s) -{ - std::vector> result; - impl->exec(s.c_str(), [&](int n, char** texts, char** names) { - std::unordered_map row; - row.reserve(n); - std::transform( - names, - names + n, - texts, - std::inserter(row, row.begin()), - [&](const char* name, const char* text) { return std::make_pair(name, text); }); - result.push_back(row); - }); - return result; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/target.cpp b/docker/rocm/migraphx/target.cpp deleted file mode 100644 index fa81d531a..000000000 --- a/docker/rocm/migraphx/target.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -void migraphx_to_value(value& v, const target& t) { v["name"] = t.name(); } -void migraphx_from_value(const value& v, target& t) -{ - t = make_target(v.at("name").to()); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/CMakeLists.txt b/docker/rocm/migraphx/targets/cpu/CMakeLists.txt deleted file mode 100644 index 558e35387..000000000 --- a/docker/rocm/migraphx/targets/cpu/CMakeLists.txt +++ /dev/null @@ -1,105 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -include(CheckCXXCompilerFlag) - -add_library(migraphx_cpu - allocate.cpp - allocation_model.cpp - binary.cpp - concat.cpp - convolution.cpp - copy.cpp - deconvolution.cpp - dnnl.cpp - eltwise.cpp - erf.cpp - fmod.cpp - fuse_ops.cpp - gather.cpp - gemm.cpp - layernorm.cpp - logsoftmax.cpp - lowering.cpp - lrn.cpp - mod.cpp - preallocate.cpp - pooling.cpp - reduction.cpp - reorder.cpp - softmax.cpp - sub.cpp - target.cpp - write_literals.cpp -) -set_target_properties(migraphx_cpu PROPERTIES EXPORT_NAME cpu) -rocm_set_soversion(migraphx_cpu ${MIGRAPHX_SO_VERSION}) - -set(MIGRAPHX_ENABLE_ZENDNN Off CACHE BOOL "") - -if(MIGRAPHX_ENABLE_ZENDNN) - find_path(ZENDNN_INC_PATH zendnn.hpp) - find_library(ZENDNN_LIB amdZenDNN) - find_library(BLIS_LIB blis) -else() - find_package(dnnl REQUIRED) -endif() - -rocm_clang_tidy_check(migraphx_cpu) -if(MIGRAPHX_ENABLE_ZENDNN) - target_compile_definitions(migraphx_cpu PRIVATE -DMIGRAPHX_ENABLE_ZENDNN) - target_include_directories(migraphx_cpu PRIVATE ${ZENDNN_INC_PATH}) - message(STATUS "ZENDNN_LIB: ${ZENDNN_LIB}") - target_link_libraries(migraphx_cpu PRIVATE ${BLIS_LIB}) - target_link_libraries(migraphx_cpu PRIVATE ${ZENDNN_LIB}) -else() - target_link_libraries(migraphx_cpu PUBLIC DNNL::dnnl) -endif() -target_link_libraries(migraphx_cpu PRIVATE migraphx) - -migraphx_generate_export_header(migraphx_cpu) - -find_package(OpenMP) -if(WIN32) - target_link_libraries(migraphx_cpu PUBLIC libomp) - target_include_directories(migraphx_cpu PUBLIC ${OpenMP_CXX_INCLUDE_DIRS}) - target_compile_options(migraphx_cpu PUBLIC ${OpenMP_CXX_FLAGS}) -else() - target_link_libraries(migraphx_cpu PUBLIC OpenMP::OpenMP_CXX) - # Add library path to rpath to workaround issues with our broken packages - foreach(LIBRARY ${OpenMP_CXX_LIBRARIES}) - if(LIBRARY MATCHES "libomp") - get_filename_component(LIBRARY_PATH "${LIBRARY}" PATH) - target_link_libraries(migraphx_cpu PUBLIC -Wl,-rpath=${LIBRARY_PATH} -Wl,-rpath-link=${LIBRARY_PATH}) - endif() - endforeach() -endif() - -rocm_install_targets( - PRIVATE - TARGETS migraphx_cpu - INCLUDE - ${CMAKE_CURRENT_SOURCE_DIR}/include -) - diff --git a/docker/rocm/migraphx/targets/cpu/allocate.cpp b/docker/rocm/migraphx/targets/cpu/allocate.cpp deleted file mode 100644 index 938139c9b..000000000 --- a/docker/rocm/migraphx/targets/cpu/allocate.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct cpu_allocate : auto_register_op -{ - shape s; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape")); - } - - std::string name() const { return "cpu::allocate"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return s; - } - argument compute(context&, const shape& output_shape, const std::vector&) const - { - argument result{output_shape}; - return result; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/allocation_model.cpp b/docker/rocm/migraphx/targets/cpu/allocation_model.cpp deleted file mode 100644 index bd6833fb9..000000000 --- a/docker/rocm/migraphx/targets/cpu/allocation_model.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -std::string cpu_allocation_model::name() const { return "cpu::allocate"; } -operation cpu_allocation_model::allocate(const shape& s) const -{ - return make_op(name(), {{"shape", to_value(s)}}); -} - -operation cpu_allocation_model::preallocate(const shape& s, const std::string& id) const -{ - return make_op("cpu::preallocate", {{"shape", to_value(s)}, {"id", id}}); -} - -std::string cpu_allocation_model::copy() const { return "cpu::copy"; } - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/binary.cpp b/docker/rocm/migraphx/targets/cpu/binary.cpp deleted file mode 100644 index e663f50e7..000000000 --- a/docker/rocm/migraphx/targets/cpu/binary.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_binary : dnnl_op -{ - std::string algo; - template - static auto reflect(Self& self, F f) - { - return pack_join(self.reflect_base(self, f), pack(f(self.algo, "algo"))); - } - - std::string group() const { return this->name() + "::" + algo; } - - std::string name() const { return "dnnl::binary"; } - - shape compute_shape(std::vector inputs) const - { - // Compensate for allocation - inputs.pop_back(); - check_shapes{this->trim_post_op_inputs(inputs), *this}.has(2); - auto s0 = inputs.at(0); - auto s1 = inputs.at(1); - auto r = s0; - if(s0 != s1 or not s0.packed()) - { - if(s0.packed() != s1.packed()) - { - r = s0.packed() ? s0 : s1; - } - else if(s0.broadcasted() != s1.broadcasted()) - { - r = s0.broadcasted() ? s1.with_lens(s0.lens()) : s0.with_lens(s0.lens()); - } - else - { - r = {s0.type(), s0.lens()}; - } - } - // Call to get_primitive to make sure an algo is available - this->get_primitive(this->to_memory_desc(r, inputs)); - return r; - } - - dnnl::binary::desc get_desc(const std::unordered_map& m) const - { - return {to_dnnl_algo(algo), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC_0)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC_1)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST))}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/concat.cpp b/docker/rocm/migraphx/targets/cpu/concat.cpp deleted file mode 100644 index 0c7cdc954..000000000 --- a/docker/rocm/migraphx/targets/cpu/concat.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_concat : dnnl_extend_op -{ - std::vector arg_map(int size) const - { - std::vector result(size); - std::iota(result.begin(), result.end(), MIGRAPHX_DNNL_PREFIX(ARG_MULTIPLE_SRC)); - return result; - } - // Custom desc class since its missing in dnnl - struct desc - { - dnnl::memory::desc dst; - std::size_t axis = 1; - std::vector srcs; - }; - desc get_desc(const std::unordered_map& m) const - { - std::vector srcs; - srcs.reserve(m.size() - 1); - - for(auto i = 0; i < m.size() - 1; i++) - { - srcs.push_back(m.at(MIGRAPHX_DNNL_PREFIX(ARG_MULTIPLE_SRC) + i)); - } - return {m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST)), std::size_t(op.axis), srcs}; - } - - auto get_primitive_desc(const desc& d, const dnnl::primitive_attr& attr) const - { - return dnnl::concat::primitive_desc(d.dst, d.axis, d.srcs, get_dnnl_context().engine, attr); - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/convolution.cpp b/docker/rocm/migraphx/targets/cpu/convolution.cpp deleted file mode 100644 index 42e533003..000000000 --- a/docker/rocm/migraphx/targets/cpu/convolution.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_convolution - : dnnl_extend_op -{ - std::vector arg_map(int) const - { - return {MIGRAPHX_DNNL_PREFIX(ARG_SRC), MIGRAPHX_DNNL_PREFIX(ARG_WEIGHTS)}; - } - - shape adjust_shape(const shape& x, int i, const shape& output) const - { - auto s = base_adjust_shape(x, output); - if(i == 1 and op.group > 1) - { - // TODO: Add support for transposed weights - if(not s.standard()) - MIGRAPHX_THROW("Weights for grouped convolution must be standard"); - auto lens = s.lens(); - lens.insert(lens.begin(), op.group); - lens.at(1) /= op.group; - return shape{s.type(), lens}; - } - return s; - } - - dnnl::convolution_forward::desc - get_desc(const std::unordered_map& m) const - { - // In DNNL dilation is zero-based - auto dilation = op.dilation; - std::transform( - dilation.begin(), dilation.end(), dilation.begin(), [](auto x) { return x - 1; }); - auto kdims = op.kdims(); - std::vector padding_l(op.padding.begin(), op.padding.begin() + kdims); - std::vector padding_r(op.padding.begin() + kdims, op.padding.end()); - return {dnnl::prop_kind::forward_inference, - dnnl::algorithm::convolution_auto, - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_WEIGHTS)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST)), - to_dnnl_dims(op.stride), - to_dnnl_dims(dilation), - to_dnnl_dims(padding_l), - to_dnnl_dims(padding_r)}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/copy.cpp b/docker/rocm/migraphx/targets/cpu/copy.cpp deleted file mode 100644 index 4c4af2b71..000000000 --- a/docker/rocm/migraphx/targets/cpu/copy.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct cpu_copy : reduce_dims_base, auto_register_op -{ - template - static auto reflect(Self&, F) - { - return pack(); - } - - std::string name() const { return "cpu::copy"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(2); - return inputs.at(1); - } - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - argument result = get_arg(args, args.size() - 1); - - visit_all(result, get_arg(args, 0))([&](auto output, auto input) { - pointwise(output, input)(ctx, output.get_shape(), 1024, [](auto& y, auto x) { y = x; }); - }); - - return result.reshape(output_shape); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/deconvolution.cpp b/docker/rocm/migraphx/targets/cpu/deconvolution.cpp deleted file mode 100644 index 3398036e1..000000000 --- a/docker/rocm/migraphx/targets/cpu/deconvolution.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_deconvolution - : dnnl_extend_op -{ - std::vector arg_map(int) const - { - return {MIGRAPHX_DNNL_PREFIX(ARG_SRC), MIGRAPHX_DNNL_PREFIX(ARG_WEIGHTS)}; - } - - shape adjust_shape(const shape& x, int i, const shape& output) const - { - auto s = base_adjust_shape(x, output); - if(i == 1) - { - // The input and output channels are flipped for dnnl - auto lens = s.lens(); - std::swap(lens[0], lens[1]); - auto strides = s.strides(); - std::swap(strides[0], strides[1]); - return {s.type(), lens, strides}; - } - return s; - } - - dnnl::deconvolution_forward::desc - get_desc(const std::unordered_map& m) const - { - // In DNNL dilation is zero-based - auto dilation = op.dilation; - std::transform( - dilation.begin(), dilation.end(), dilation.begin(), [](auto x) { return x - 1; }); - return {dnnl::prop_kind::forward_inference, - dnnl::algorithm::deconvolution_direct, - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_WEIGHTS)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST)), - to_dnnl_dims(op.stride), - to_dnnl_dims(dilation), - to_dnnl_dims(op.padding), - to_dnnl_dims(op.padding)}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/dnnl.cpp b/docker/rocm/migraphx/targets/cpu/dnnl.cpp deleted file mode 100644 index dc252cdfe..000000000 --- a/docker/rocm/migraphx/targets/cpu/dnnl.cpp +++ /dev/null @@ -1,205 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include - -#if defined(__GNUC__) && __GNUC__ <= 5 -namespace std { -#ifdef MIGRAPHX_ENABLE_ZENDNN -namespace dnnl = zendnn; -#endif -template <> -struct hash -{ - using argument_type = dnnl::algorithm; - using result_type = std::size_t; - result_type operator()(const argument_type& x) const noexcept - { - return std::hash>{}( - static_cast>(x)); - } -}; - -} // namespace std -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -dnnl_context& get_dnnl_context() -{ - static dnnl_context ctx{}; // NOLINT - return ctx; -} - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wswitch-enum" -#endif -dnnl::memory::data_type to_dnnl_memory_data_type(shape::type_t t) -{ - using dt = dnnl::memory::data_type; - using st = shape::type_t; - switch(t) - { - case st::half_type: return dt::f16; - case st::float_type: return dt::f32; - case st::int32_type: return dt::s32; - case st::int8_type: return dt::s8; - case st::uint8_type: return dt::u8; - case st::fp8e4m3fnuz_type: MIGRAPHX_THROW("fp8e4m3fnuz unsupported in DNNL"); - default: MIGRAPHX_THROW("Unsupported data type"); - } -} -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -dnnl::memory::format_tag to_dnnl_memory_format_tag(std::size_t n) -{ - switch(n) - { - case 1: return dnnl::memory::format_tag::a; - case 2: return dnnl::memory::format_tag::ab; - case 3: return dnnl::memory::format_tag::abc; - case 4: return dnnl::memory::format_tag::abcd; - case 5: return dnnl::memory::format_tag::abcde; - case 6: return dnnl::memory::format_tag::abcdef; - default: MIGRAPHX_THROW("Unsupported tensor size: " + std::to_string(n)); - } -} - -dnnl::memory::desc to_dnnl_memory_desc(const shape& s) -{ - return {to_dnnl_dims(s.lens()), to_dnnl_memory_data_type(s.type()), to_dnnl_dims(s.strides())}; -} - -dnnl::memory to_dnnl_memory(const dnnl::memory::desc& desc, const argument& a) -{ - return {desc, get_dnnl_context().engine, a.data()}; -} - -dnnl::memory to_dnnl_memory(const argument& a) -{ - return to_dnnl_memory(to_dnnl_memory_desc(a.get_shape()), a); -} - -// clang-format off -#define MIGRAPHX_VISIT_DNNL_ALGO(m) \ - m(undef) \ - m(convolution_auto) \ - m(convolution_direct) \ - m(convolution_winograd) \ - m(deconvolution_direct) \ - m(deconvolution_winograd) \ - m(eltwise_relu) \ - m(eltwise_tanh) \ - m(eltwise_elu) \ - m(eltwise_square) \ - m(eltwise_abs) \ - m(eltwise_sqrt) \ - m(eltwise_swish) \ - m(eltwise_linear) \ - m(eltwise_bounded_relu) \ - m(eltwise_soft_relu) \ - m(eltwise_logistic) \ - m(eltwise_exp) \ - m(eltwise_gelu) \ - m(eltwise_gelu_tanh) \ - m(eltwise_gelu_erf) \ - m(eltwise_log) \ - m(eltwise_clip) \ - m(eltwise_pow) \ - m(eltwise_round) \ - m(eltwise_relu_use_dst_for_bwd) \ - m(eltwise_tanh_use_dst_for_bwd) \ - m(eltwise_elu_use_dst_for_bwd) \ - m(eltwise_sqrt_use_dst_for_bwd) \ - m(eltwise_logistic_use_dst_for_bwd) \ - m(eltwise_exp_use_dst_for_bwd) \ - m(lrn_across_channels) \ - m(lrn_within_channel) \ - m(pooling_max) \ - m(pooling_avg) \ - m(pooling_avg_include_padding) \ - m(pooling_avg_exclude_padding) \ - m(vanilla_rnn) \ - m(vanilla_lstm) \ - m(vanilla_gru) \ - m(lbr_gru) \ - m(binary_add) \ - m(binary_mul) \ - m(binary_max) \ - m(binary_min) \ - m(binary_div) \ - m(resampling_nearest) \ - m(resampling_linear) \ - m(reduction_max) \ - m(reduction_min) \ - m(reduction_sum) \ - m(reduction_mul) \ - m(reduction_mean) \ - m(reduction_norm_lp_max) \ - m(reduction_norm_lp_sum) \ - m(reduction_norm_lp_power_p_max) \ - m(reduction_norm_lp_power_p_sum) -// clang-format on - -const std::unordered_map& dnnl_algo_map() -{ - static const std::unordered_map m = { -#define MIGRAPHX_DNNL_ALGO_GENERATE_VISITOR(x) {#x, dnnl::algorithm::x}, - MIGRAPHX_VISIT_DNNL_ALGO(MIGRAPHX_DNNL_ALGO_GENERATE_VISITOR) -#undef MIGRAPHX_DNNL_ALGO_GENERATE_VISITOR - }; - return m; -} - -dnnl::algorithm to_dnnl_algo(const std::string& name) -{ - if(dnnl_algo_map().count(name) == 0) - MIGRAPHX_THROW("Missing dnnl algo: " + name); - return dnnl_algo_map().at(name); -} - -const std::unordered_map& dnnl_algo_string_map() -{ - static const std::unordered_map m = { -#define MIGRAPHX_DNNL_ALGO_GENERATE_VISITOR(x) {dnnl::algorithm::x, #x}, - MIGRAPHX_VISIT_DNNL_ALGO(MIGRAPHX_DNNL_ALGO_GENERATE_VISITOR) -#undef MIGRAPHX_DNNL_ALGO_GENERATE_VISITOR - }; - return m; -} - -std::string to_string(const dnnl::algorithm& algo) -{ - if(dnnl_algo_string_map().count(algo) == 0) - return "unknown_" + std::to_string(static_cast(algo)); - return dnnl_algo_string_map().at(algo); -} - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/eltwise.cpp b/docker/rocm/migraphx/targets/cpu/eltwise.cpp deleted file mode 100644 index 5b328cb7e..000000000 --- a/docker/rocm/migraphx/targets/cpu/eltwise.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_eltwise : dnnl_op -{ - std::string algo; - float alpha = 0; - float beta = 0; - template - static auto reflect(Self& self, F f) - { - return pack_join(self.reflect_base(self, f), - pack(f(self.algo, "algo"), f(self.alpha, "alpha"), f(self.beta, "beta"))); - } - - std::string group() const { return this->name() + "::" + algo; } - - std::string name() const { return "dnnl::eltwise"; } - - shape compute_shape(std::vector inputs) const - { - // Compensate for allocation - inputs.pop_back(); - check_shapes{this->trim_post_op_inputs(inputs), *this}.has(1).packed(); - auto s = inputs.at(0); - auto r = s; - if(not s.packed()) - r = shape{s.type(), s.lens()}; - // Call to get_primitive to make sure an algo is available - this->get_primitive(this->to_memory_desc(r, inputs)); - return r; - } - - dnnl::eltwise_forward::desc get_desc(const std::unordered_map& m) const - { - return {dnnl::prop_kind::forward_inference, - to_dnnl_algo(algo), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC_0)), - alpha, - beta}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/erf.cpp b/docker/rocm/migraphx/targets/cpu/erf.cpp deleted file mode 100644 index 9fa34b4fa..000000000 --- a/docker/rocm/migraphx/targets/cpu/erf.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -template struct cpu_unary; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/fmod.cpp b/docker/rocm/migraphx/targets/cpu/fmod.cpp deleted file mode 100644 index ade453147..000000000 --- a/docker/rocm/migraphx/targets/cpu/fmod.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -template struct cpu_binary; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/fuse_ops.cpp b/docker/rocm/migraphx/targets/cpu/fuse_ops.cpp deleted file mode 100644 index a4f8fe78f..000000000 --- a/docker/rocm/migraphx/targets/cpu/fuse_ops.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_DNNL_POST_OPS_WORKAROUND); - -MIGRAPHX_PRED_MATCHER(has_post_ops, instruction_ref ins) -{ - auto v = ins->get_operator().to_value(); - return v.contains("post_ops"); -} - -MIGRAPHX_PRED_MATCHER(without_post_ops, instruction_ref ins) -{ - auto v = ins->get_operator().to_value(); - return v.contains("post_ops") and v["post_ops"].empty(); -} - -bool workaround_dnnl_broken_post_ops(const operation& op, const operation& post_op) -{ - if(contains({"dnnl::dot", "dnnl::convolution"}, op.name())) - return true; - auto pv = post_op.to_value(); - if(not pv.at("post_ops").empty()) - return true; - auto v = op.to_value(); - auto last_op = v.at("post_ops").empty() ? v : v.at("post_ops").back(); - auto algo = last_op.contains("algo") ? last_op.at("algo").to() : op.name(); - auto post_algo = pv["algo"].to(); - if(starts_with(algo, "eltwise") and starts_with(post_algo, "eltwise")) - return true; - if(algo == post_algo) - return true; - return false; -} - -operation merge_post_ops(const operation& op, const operation& post_op) -{ - auto pv = post_op.to_value(); - auto v = op.to_value(); - v["post_ops"].push_back({{"algo", pv["algo"]}, - {"alpha", pv["alpha"].value_or(0.0f)}, - {"beta", pv["beta"].value_or(0.0f)}}); - auto post_ops = pv.at("post_ops"); - for(const auto& po : post_ops) - v["post_ops"].push_back(po); - return make_op(op.name(), v); -} - -struct find_post_ops -{ - context* ctx = nullptr; - match::any_matcher matcher() const - { - if(enabled(MIGRAPHX_DISABLE_DNNL_POST_OPS_WORKAROUND{})) - return match::name("dnnl::eltwise", - "dnnl::binary")(match::arg(0)(has_post_ops(), match::used_once())); - else - { - auto dnnl_binary = match::name("dnnl::binary")(without_post_ops(), match::used_once()); - return match::name("dnnl::eltwise")(without_post_ops(), match::arg(0)(dnnl_binary)); - } - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = ins->inputs().front(); - auto x = x_ins->get_operator(); - - if(workaround_dnnl_broken_post_ops(x, ins->get_operator())) - return; - - auto op = merge_post_ops(x, ins->get_operator()); - auto inputs = x_ins->inputs(); - inputs.back() = ins->inputs().back(); - if(ins->name() == "dnnl::binary") - inputs.insert(std::prev(inputs.end()), ins->inputs().at(1)); - auto input_shapes = to_shapes(inputs); - auto new_shape = try_compute_shape(op, input_shapes); - if(new_shape.empty() or new_shape.front() != ins->get_shape()) - return; - auto info = compile(op, *ctx, new_shape.front(), input_shapes); - if(info.contains("impl") and starts_with(info.at("impl").to(), "ref:")) - return; - m.replace_instruction(ins, op, inputs); - } -}; - -void fuse_ops::apply(module& m) const -{ - for(std::size_t i = 0; i < 4; i++) - { - match::find_matches(m, find_post_ops{ctx}); - dead_code_elimination{}.apply(m); - } -} - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/gather.cpp b/docker/rocm/migraphx/targets/cpu/gather.cpp deleted file mode 100644 index 40bc556b9..000000000 --- a/docker/rocm/migraphx/targets/cpu/gather.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct cpu_gather : auto_register_op -{ - op::gather op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - std::string name() const { return "cpu::" + op.name(); } - shape compute_shape(std::vector inputs) const - { - // Compensate for allocation - inputs.pop_back(); - check_shapes(inputs, *this).standard(); - return migraphx::compute_shape(op, inputs); - } - - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - std::size_t nelements = output_shape.elements(); - auto lens = args[0].get_shape().lens(); - auto axis_dim_size = lens[op.axis]; - lens[op.axis] = args[1].get_shape().elements(); - shape out_comp{output_shape.type(), lens}; - - visit_all(args.back(), args[0])([&](auto output, auto input) { - args[1].visit([&](auto indices) { - const auto* indices_ptr = indices.data(); - auto* output_ptr = output.data(); - ctx.bulk_execute(nelements, 1024, [=](auto start, auto end) { - for(auto i = start; i < end; i++) - { - auto idx = out_comp.multi(i); - auto in_index = indices_ptr[idx[op.axis]]; - in_index = (in_index < 0) ? in_index + axis_dim_size : in_index; - idx[op.axis] = in_index; - output_ptr[i] = input(idx.begin(), idx.end()); - } - }); - }); - }); - - return args.back(); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/gemm.cpp b/docker/rocm/migraphx/targets/cpu/gemm.cpp deleted file mode 100644 index 50f42d5fe..000000000 --- a/docker/rocm/migraphx/targets/cpu/gemm.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_gemm : dnnl_extend_op -{ - std::vector arg_map(int) const - { - return {MIGRAPHX_DNNL_PREFIX(ARG_SRC), - MIGRAPHX_DNNL_PREFIX(ARG_WEIGHTS), - MIGRAPHX_DNNL_PREFIX(ARG_BIAS)}; - } - - template - void required(const check_shapes& cs) const - { - cs.not_broadcasted(); - } - - dnnl::matmul::desc get_desc(const std::unordered_map& m) const - { - return {m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_WEIGHTS)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST))}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/allocation_model.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/allocation_model.hpp deleted file mode 100644 index 4ee101331..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/allocation_model.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_ALLOCATION_MODEL_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_ALLOCATION_MODEL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct cpu_allocation_model -{ - std::string name() const; - std::string copy() const; - operation allocate(const shape& s) const; - operation preallocate(const shape& s, const std::string& id) const; - bool needs_out_params() const { return false; } -}; - -} // namespace cpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/context.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/context.hpp deleted file mode 100644 index 461dbcb39..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/context.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CONTEXT_HPP -#define MIGRAPHX_GUARD_RTGLIB_CONTEXT_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct context -{ - void finish() const {} - - template - void bulk_execute(std::size_t n, std::size_t min_grain, F f) - { - cpu::parallel_for(n, min_grain, f); - } - - template - void bulk_execute(std::size_t n, F f) - { - this->bulk_execute(n, 256, f); - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/dnnl.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/dnnl.hpp deleted file mode 100644 index b05cad852..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/dnnl.hpp +++ /dev/null @@ -1,441 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_DNNL_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_DNNL_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef MIGRAPHX_ENABLE_ZENDNN -#include -#else -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -#ifdef MIGRAPHX_ENABLE_ZENDNN -namespace dnnl = zendnn; -#define MIGRAPHX_CONCAT_PREFIX(b) ZENDNN_##b // NOLINT -#else -#define MIGRAPHX_CONCAT_PREFIX(b) DNNL_##b // NOLINT -#endif -#define MIGRAPHX_DNNL_PREFIX(b) MIGRAPHX_CONCAT_PREFIX(b) // NOLINT - -struct dnnl_context -{ - dnnl::engine engine; - dnnl::stream stream; - dnnl_context() : engine(dnnl::engine::kind::cpu, 0), stream(engine) {} -}; - -dnnl_context& get_dnnl_context(); - -dnnl::memory::data_type to_dnnl_memory_data_type(shape::type_t t); - -dnnl::memory::format_tag to_dnnl_memory_format_tag(std::size_t n); - -template -inline dnnl::memory::dims to_dnnl_dims(R&& r) -{ - return {r.begin(), r.end()}; -} - -dnnl::memory::desc to_dnnl_memory_desc(const shape& s); - -dnnl::memory to_dnnl_memory(const dnnl::memory::desc& desc, const argument& a); - -dnnl::memory to_dnnl_memory(const argument& a); - -dnnl::algorithm to_dnnl_algo(const std::string& name); - -std::string to_string(const dnnl::algorithm& algo); - -struct post_op : reflect_equality, reflect_stream -{ - std::string algo; - float alpha = 0; - float beta = 0; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.algo, "algo"), f(self.alpha, "alpha"), f(self.beta, "beta")); - } -}; - -template -struct execute_wrapper -{ - F f; - argument operator()(context&, const std::vector& args) const { return f(args); } -}; - -template -execute_wrapper make_execute_wrapper(F f) -{ - return {std::move(f)}; -} - -template -struct dnnl_op : auto_register_op -{ - std::vector post_ops; - std::function& args)> execute; - - template - static auto reflect_base(Self& self, F f) - { - return pack(f(self.post_ops, "post_ops")); - } - - template - static auto reflect(Self& self, F f) - { - return reflect_base(self, f); - } - - std::string group() const - { - const auto& self = static_cast(*this); - return self.name(); - } - - value attributes() const - { - std::vector names; - std::transform(post_ops.begin(), post_ops.end(), std::back_inserter(names), [](auto&& op) { - return op.algo; - }); - const auto& self = static_cast(*this); - auto g = self.group(); - if(not names.empty()) - g += "<" + join_strings(names, ",") + ">"; - return {{"group", g}}; - } - - std::size_t get_extra_post_op_args() const - { - return std::count_if(post_ops.begin(), post_ops.end(), [](const auto& po) { - return contains(po.algo, "binary"); - }); - } - - static std::size_t get_binary_post_op_arg(std::size_t pos) - { - return MIGRAPHX_DNNL_PREFIX(ARG_ATTR_MULTIPLE_POST_OP)(pos) | // NOLINT - MIGRAPHX_DNNL_PREFIX(ARG_SRC_1); // NOLINT - } - - static std::vector to_shapes(const std::vector& args) - { - std::vector shapes(args.size()); - std::transform(args.begin(), args.end(), shapes.begin(), [](const argument& a) { - return a.get_shape(); - }); - return shapes; - } - static std::string impl(const Primitive& prim) - { - auto desc = prim.get_primitive_desc(); - const char* str = nullptr; -#ifdef MIGRAPHX_ENABLE_ZENDNN - zendnn_primitive_desc_query( - desc, zendnn_query_impl_info_str, 0, reinterpret_cast(&str)); -#else - dnnl_primitive_desc_query(desc, dnnl_query_impl_info_str, 0, reinterpret_cast(&str)); -#endif - return str == nullptr ? "" : str; - } - // Map arg index to arg in dnnl - std::vector arg_map(int size) const - { - std::vector result(size); - std::iota(result.begin(), result.end(), MIGRAPHX_DNNL_PREFIX(ARG_SRC_0)); - return result; - } - shape base_adjust_shape(const shape& s, const shape& output) const - { - if(s.broadcasted()) - { - auto lens = s.lens(); - auto strides = s.strides(); - std::transform(strides.begin(), - strides.end(), - lens.begin(), - lens.begin(), - [](auto stride, auto len) -> std::size_t { - if(stride == 0) - return 1; - else - return len; - }); - // Use the permutation of the output - return output.with_lens(s.type(), lens); - } - return s; - } - template - void for_each_post_op(F f) const - { - int i = 0; - for(auto&& op : post_ops) - { - if(contains(op.algo, "binary")) - { - f(op, get_binary_post_op_arg(i)); - } - else - { - f(op, -1); - } - i++; - } - } - shape adjust_shape(const shape& s, int, const shape& output) const - { - return base_adjust_shape(s, output); - } - std::vector create_arg_map(std::size_t input_size) const - { - const auto& self = static_cast(*this); - auto npost_ops = get_extra_post_op_args(); - auto prim_input_size = input_size - npost_ops; - auto m = self.arg_map(prim_input_size); - for_each_post_op([&](auto&&, auto arg) { - if(arg < 0) - return; - m.push_back(arg); - }); - return m; - } - std::unordered_map - to_memory_desc(const shape& output_shape, const std::vector& inputs) const - { - const auto& self = static_cast(*this); - std::unordered_map result; - result[MIGRAPHX_DNNL_PREFIX(ARG_DST)] = - to_dnnl_memory_desc(self.adjust_shape(output_shape, inputs.size(), output_shape)); - auto m = create_arg_map(inputs.size()); - assert(m.size() >= inputs.size()); - for(int i = 0; i < inputs.size(); i++) - { - result[m[i]] = to_dnnl_memory_desc(self.adjust_shape(inputs[i], i, output_shape)); - } - return result; - } - dnnl::primitive_attr - get_primitive_attr(const std::unordered_map& m) const - { - dnnl::primitive_attr result; - dnnl::post_ops po; - for_each_post_op([&](auto&& op, auto arg) { - if(contains(op.algo, "binary_add")) - { - auto desc = m.at(arg); - if(desc == m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST))) - po.append_sum(1.0f); - else - po.append_binary(to_dnnl_algo(op.algo), m.at(arg)); - } - else if(contains(op.algo, "binary")) - { - po.append_binary(to_dnnl_algo(op.algo), m.at(arg)); - } - else if(contains(op.algo, "eltwise")) - po.append_eltwise(1.0f, to_dnnl_algo(op.algo), op.alpha, op.beta); - else - MIGRAPHX_THROW("Unknown post op algo: " + op.algo); - }); - result.set_post_ops(po); - return result; - } - template - auto get_primitive_desc(const T& desc, const dnnl::primitive_attr& attr) const - -> decltype(typename Primitive::primitive_desc(desc, attr, get_dnnl_context().engine)) - { - return typename Primitive::primitive_desc(desc, attr, get_dnnl_context().engine); - } - Primitive get_primitive(const std::unordered_map& m) const - { - const auto& self = static_cast(*this); - auto desc = self.get_desc(m); - auto attr = MIGRAPHX_ASSERT_NO_THROW(this->get_primitive_attr(m)); - auto pd = self.get_primitive_desc(desc, attr); - return Primitive(pd); - } - argument compute(context& ctx, const shape&, const std::vector& args) const - { - return execute(ctx, args); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } - value compile(context&, const shape& output_shape, std::vector inputs) - { - // Compensate for allocation - inputs.pop_back(); - auto md = to_memory_desc(output_shape, inputs); - auto prim = get_primitive(md); - auto impl_name = impl(prim); - return {{"impl", impl_name}}; - } - - void finalize(context&, const shape& output_shape, std::vector inputs) - { - // Compensate for allocation - inputs.pop_back(); - const auto& self = static_cast(*this); - auto name = self.name(); - auto md = to_memory_desc(output_shape, inputs); - auto prim = get_primitive(md); - auto arg_lookup = create_arg_map(inputs.size()); -#ifndef NDEBUG - auto prim_attr = get_primitive_attr(md); -#endif - execute = make_execute_wrapper([=](const std::vector& args) { -#ifndef NDEBUG - // Check that the memory descriptors have not changed - auto debug_args = args; - debug_args.pop_back(); - auto debug_md = to_memory_desc(output_shape, to_shapes(debug_args)); - for(auto&& p : debug_md) - { - if(md.count(p.first) == 0) - MIGRAPHX_THROW(name + - ": Missing memory descriptor for: " + std::to_string(p.first)); - if(p.second == md.at(p.first)) - continue; - MIGRAPHX_THROW(name + - ": Memory descriptor has changed for: " + std::to_string(p.first)); - } - // Check post_ops args are correct - auto pos = prim_attr.get_post_ops(); - auto prim_input_size = inputs.size() - this->get_extra_post_op_args(); - int j = 0; - for(int i = 0; i < pos.len(); i++) - { - auto arg = j + prim_input_size; - auto kind = pos.kind(i); - std::string mesg = - "Post op " + std::to_string(i) + "@" + std::to_string(arg) + ": "; - try - { - dnnl::algorithm algo; - dnnl::memory::desc mdesc; - float scale = 0; - float alpha = 0; - float beta = 0; - if(kind == dnnl::primitive::kind::binary) - { - pos.get_params_binary(i, algo, mdesc); - if(mdesc != md.at(arg_lookup.at(arg))) - MIGRAPHX_THROW(mesg + - "Memory descriptor doesn't match for binary post op"); - j++; - } - else if(kind == dnnl::primitive::kind::eltwise) - { - pos.get_params_eltwise(i, scale, algo, alpha, beta); - } - else if(kind == dnnl::primitive::kind::sum) - { - pos.get_params_sum(i, scale); - algo = dnnl::algorithm::binary_add; - } - else - { - MIGRAPHX_THROW("Unknown kind"); - } - if(to_dnnl_algo(post_ops[i].algo) != algo) - MIGRAPHX_THROW(mesg + "Algorithm doesn't match for post op " + - post_ops[i].algo + " != " + to_string(algo)); - } - catch(const dnnl::error& e) - { - MIGRAPHX_THROW(mesg + "Failed to get post ops argument " + ": " + e.what()); - } - } -#endif - std::unordered_map m; - m[MIGRAPHX_DNNL_PREFIX(ARG_DST)] = - to_dnnl_memory(md.at(MIGRAPHX_DNNL_PREFIX(ARG_DST)), args.back()); - for(int i = 0; i < args.size() - 1; i++) - m[arg_lookup[i]] = to_dnnl_memory(md.at(arg_lookup[i]), args[i]); - prim.execute(get_dnnl_context().stream, m); - return args.back(); - }); - } - std::vector trim_post_op_inputs(const std::vector& inputs) const - { - auto prim_input_size = inputs.size() - this->get_extra_post_op_args(); - return {inputs.begin(), inputs.begin() + prim_input_size}; - } -}; - -template -struct dnnl_extend_op : dnnl_op -{ - Op op; - - template - static auto reflect(Self& self, F f) - { - return pack_join(self.reflect_base(self, f), migraphx::reflect(self.op, f)); - } - - // dnnl has some issues with non-packed inputs - template - void required(const check_shapes& cs) const - { - cs.packed_or_broadcasted(); - } - - std::string name() const { return "dnnl::" + op.name(); } - shape compute_shape(std::vector inputs) const - { - const auto& self = static_cast(*this); - // Compensate for allocation - inputs.pop_back(); - self.required(check_shapes(inputs, self)); - auto r = migraphx::compute_shape(op, this->trim_post_op_inputs(inputs)); - // Call to get_primitive to make sure an algo is available - this->get_primitive(this->to_memory_desc(r, inputs)); - return r; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/fuse_ops.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/fuse_ops.hpp deleted file mode 100644 index e0918846a..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/fuse_ops.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_CPU_FUSE_OPS_HPP -#define MIGRAPHX_GUARD_CPU_FUSE_OPS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace cpu { - -struct MIGRAPHX_CPU_EXPORT fuse_ops -{ - context* ctx = nullptr; - std::string name() const { return "cpu::fuse_ops"; } - void apply(module& m) const; -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_CPU_FUSE_OPS_HPP diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/lowering.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/lowering.hpp deleted file mode 100644 index d4b96c543..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/lowering.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CPU_LOWERING_HPP -#define MIGRAPHX_GUARD_RTGLIB_CPU_LOWERING_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace cpu { - -struct MIGRAPHX_CPU_EXPORT lowering -{ - std::string name() const { return "cpu::lowering"; } - void apply(module& m) const; -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/parallel.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/parallel.hpp deleted file mode 100644 index cb3b9ed64..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/parallel.hpp +++ /dev/null @@ -1,125 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_PARALLEL_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_PARALLEL_HPP - -// #define MIGRAPHX_DISABLE_OMP -#include -#include -#include -#ifdef MIGRAPHX_DISABLE_OMP -#include -#else - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreserved-identifier" -#endif -#include -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -#ifdef MIGRAPHX_DISABLE_OMP - -inline std::size_t max_threads() { return std::thread::hardware_concurrency(); } - -template -void parallel_for_impl(std::size_t n, std::size_t threadsize, F f) -{ - if(threadsize <= 1) - { - f(std::size_t{0}, n); - } - else - { - std::vector threads(threadsize); -// Using const here causes gcc 5 to ICE -#if(!defined(__GNUC__) || __GNUC__ != 5) - const -#endif - std::size_t grainsize = std::ceil(static_cast(n) / threads.size()); - - std::size_t work = 0; - std::generate(threads.begin(), threads.end(), [=, &work] { - auto result = joinable_thread([=]() mutable { - assert(work < n); - f(work, std::min(n, work + grainsize)); - }); - work += grainsize; - return result; - }); - // cppcheck-suppress unsignedLessThanZero - assert(work >= n); - } -} -#else - -inline std::size_t max_threads() { return omp_get_max_threads(); } - -template -void parallel_for_impl(std::size_t n, std::size_t threadsize, F f) -{ - if(threadsize <= 1) - { - f(std::size_t{0}, n); - } - else - { - std::size_t grainsize = std::ceil(static_cast(n) / threadsize); -#pragma omp parallel for num_threads(threadsize) schedule(static, 1) - for(std::size_t tid = 0; tid < threadsize; tid++) - { - std::size_t work = tid * grainsize; - assert(work < n); - f(work, std::min(n, work + grainsize)); - } - } -} -#endif -template -void parallel_for(std::size_t n, std::size_t min_grain, F f) -{ - const auto threadsize = std::min(max_threads(), n / min_grain); - parallel_for_impl(n, threadsize, f); -} - -template -void parallel_for(std::size_t n, F f) -{ - const int min_grain = 8; - parallel_for(n, min_grain, f); -} - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/pointwise.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/pointwise.hpp deleted file mode 100644 index ece5498c8..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/pointwise.hpp +++ /dev/null @@ -1,414 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_POINTWISE_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_POINTWISE_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct multi_index -{ - constexpr multi_index() = default; - - multi_index(const shape& s, std::size_t i) : n(s.lens().size()) - { - assert(n < max_size); - std::copy(s.lens().begin(), s.lens().end(), dims); - s.multi_copy(i, index, index + max_size); - } - - constexpr std::size_t size() const { return n; } - - constexpr std::size_t* begin() { return index; } - constexpr const std::size_t* begin() const { return index; } - - constexpr std::size_t* end() { return index + size(); } - constexpr const std::size_t* end() const { return index + size(); } - - std::size_t offset(const shape& s) const { return s.index(begin(), end()); } - - constexpr void carry() - { - std::size_t overflow = 0; - for(std::ptrdiff_t i = size() - 1; i > 0; i--) - { - auto z = index[i] + overflow; - // Reset overflow - overflow = 0; - // Compute overflow using while loop instead of mod - // overflow = z / dims[i]; - // z = z % dims[i]; - while(z >= dims[i]) - { - z -= dims[i]; - overflow += 1; - } - index[i] = z; - // Exit if there is no overflow - if(overflow == 0) - return; - } - index[0] += overflow; - } - - constexpr void increment(std::size_t i) - { - index[size() - 1] += i; - carry(); - } - - constexpr multi_index& operator+=(std::size_t i) - { - increment(i); - return *this; - } - - constexpr multi_index& operator++() - { - increment(1); - return *this; - } - multi_index operator++(int) // NOLINT - { - multi_index result = *this; - increment(1); - return result; - } - - private: - static const std::size_t max_size = 5; - std::size_t index[max_size] = {}; - std::size_t dims[max_size] = {}; - std::size_t n = 0; -}; - -struct reduce_dims_base -{ - std::vector reduce_shapes; - - void finalize(context&, const shape&, const std::vector& inputs) - { - reduce_shapes = reduce_dims(inputs); - } - - argument get_arg(const std::vector& args, std::size_t i) const - { - if(reduce_shapes.empty()) - return args[i]; - return args.at(i).reshape(reduce_shapes.at(i)); - } - - argument get_output() const - { - argument a{reduce_shapes[0]}; - return a; - } -}; - -template -struct vec -{ - using array_type = std::array; - using vector_type __attribute__((vector_size(N * sizeof(T)))) = T; - union - { - array_type array; - vector_type vector; - }; - - static_assert(sizeof(array_type) == sizeof(vector_type), "Not the same size"); -}; - -template -constexpr std::integral_constant vec_size(const T&) -{ - return {}; -} - -template -constexpr std::integral_constant vec_size(const vec&) -{ - return {}; -} - -template -constexpr std::size_t vec_size() -{ - return decltype(vec_size(std::declval())){}; -} - -template () > 0))> -void vec_apply(F f, V& v, Vs... vs) -{ - assert(all_of({vec_size()...}, [&](auto n) { return n == vec_size(); })); - assert(vec_size() == v.array.size()); - for(std::size_t i = 0; i < vec_size(); i++) - f(v.array[i], vs.vector[i]...); -} - -template () == 0))> -void vec_apply(F f, V& v, Vs&... vs) -{ - f(v, vs...); -} - -inline std::size_t find_packed_len(const shape& s) -{ - for(std::size_t i = 0; i < s.lens().size(); i++) - { - if(s.lens()[i] > 1 and s.strides()[i] == 1) - { - return i; - } - } - return -1; -} - -template -shape vectorize(const shape& s) -{ - assert(s.standard() or s.broadcasted()); - auto lens = s.lens(); - if(s.broadcasted()) - { - auto n = find_packed_len(s); - assert(n != -1); - assert((lens[n] % N) == 0); - lens[n] /= N; - return {s.type(), lens, s.strides()}; - } - assert((lens.back() % N) == 0); - lens.back() /= N; - return {s.type(), lens}; -} - -template -tensor_view> vectorize(tensor_view tv) -{ - return {vectorize(tv.get_shape()), reinterpret_cast*>(tv.data())}; -} - -template -struct is_vector_type : std::false_type -{ -}; - -template <> -struct is_vector_type : std::true_type -{ -}; - -template -struct is_vector_tensor_view : and_{}...> -{ -}; - -template -bool is_vectorizable(const Xs&... xs) -{ - return all_of({xs...}, [](const auto& s) { - if(s.standard() and (s.lens().back() % N) == 0) - return true; - if(s.broadcasted()) - { - auto n = std::inner_product(s.lens().begin(), - s.lens().end(), - s.strides().begin(), - 0, - std::plus<>{}, - [&](auto len, auto stride) -> std::size_t { - if(stride > 0 and len == 1) - return 0; - return stride; - }); - if(n == 1) - { - auto i = find_packed_len(s); - assert(i != -1); - return (s.lens()[i] % N) == 0; - } - } - return false; - }); -} - -template {})> -auto auto_vectorize(const shape& base_shape, Ts... xs) -{ - return [=](auto f) { - if(is_vectorizable<32>(base_shape, xs.get_shape()...)) - f(vectorize<32>(base_shape), vectorize<32>(xs)...); - else if(is_vectorizable<8>(base_shape, xs.get_shape()...)) - f(vectorize<8>(base_shape), vectorize<8>(xs)...); - else - f(base_shape, xs...); - }; -} - -template {})> -auto auto_vectorize(const shape& base_shape, Ts... xs) -{ - return [=](auto f) { f(base_shape, xs...); }; -} - -template -bool is_standard_offset(const X& x, const Xs&... xs) -{ - if(all_of({x, xs...}, [](const auto& s) { return s.standard(); })) - return true; - if(all_of({x, xs...}, [](const auto& s) { return s.packed(); }) and - all_of({xs...}, [&](const auto& s) { return s == x; })) - return true; - return false; -} - -template -auto pointwise_apply(Ts... ts) -{ - return [=](context& ctx, const shape& base_shape, std::size_t min_grain, auto f) mutable { - if(is_standard_offset(ts.get_shape()...)) - { - ctx.bulk_execute(base_shape.elements(), min_grain, [=](auto start, auto end) mutable { - for(auto i = start; i < end; i++) - { - vec_apply(f, ts.data()[i]...); - } - }); - } - else - { - assert(base_shape.lens().size() <= 6); - ctx.bulk_execute(base_shape.elements(), min_grain, [=](auto start, auto end) mutable { - multi_index mi(base_shape, start); - for(auto i = start; i < end; i++) - { - vec_apply(f, ts.data()[mi.offset(ts.get_shape())]...); - ++mi; - } - }); - } - }; -} - -template -auto pointwise(Ts... ts) -{ - return [=](context& ctx, const shape& base_shape, std::size_t min_grain, auto f) mutable { - auto_vectorize(base_shape, ts...)( - [&](auto bs, auto... xs) { pointwise_apply(xs...)(ctx, bs, min_grain, f); }); - }; -} - -template -struct cpu_unary : reduce_dims_base, auto_register_op> -{ - Op op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - std::string name() const { return "cpu::" + op.name(); } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(2); - const auto& s = inputs.at(0); - return {s.type(), s.lens()}; - } - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - argument result = get_arg(args, args.size() - 1); - - visit_all(result, get_arg(args, 0))([&](auto output, auto input) { - auto op2 = op; - pointwise(output, input)( - ctx, output.get_shape(), 1024, [op2](auto& y, auto x) { y = op2.apply()(x); }); - }); - - return result.reshape(output_shape); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -template -struct cpu_binary : reduce_dims_base, auto_register_op> -{ - Op op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - std::string name() const { return "cpu::" + op.name(); } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(3); - const auto& s = inputs.at(0); - return {s.type(), s.lens()}; - } - - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - argument result = get_arg(args, args.size() - 1); - - visit_all(result, get_arg(args, 0), get_arg(args, 1))( - [&](auto output, auto input1, auto input2) { - auto op2 = op; - pointwise(output, input1, input2)( - ctx, output.get_shape(), 1024, [op2](auto& z, auto x, auto y) { - z = op2.apply()(x, y); - }); - }); - - return result.reshape(output_shape); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/target.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/target.hpp deleted file mode 100644 index 589b680fe..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/target.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_CPU_TARGET_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_CPU_TARGET_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct pass; -namespace cpu { - -struct MIGRAPHX_CPU_EXPORT target -{ - std::string name() const; - std::vector get_passes(migraphx::context& gctx, const compile_options&) const; - migraphx::context get_context() const { return context{}; } - argument copy_to(const argument& arg) const { return arg; } - argument copy_from(const argument& arg) const { return arg; } - argument allocate(const shape& s) const; -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/write_literals.hpp b/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/write_literals.hpp deleted file mode 100644 index 3c23fb14f..000000000 --- a/docker/rocm/migraphx/targets/cpu/include/migraphx/cpu/write_literals.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_WRITE_LITERALS_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_CPU_WRITE_LITERALS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct module; -namespace cpu { - -struct write_literals -{ - std::string name() const { return "cpu::write_literals"; } - void apply(module& m) const; -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/cpu/layernorm.cpp b/docker/rocm/migraphx/targets/cpu/layernorm.cpp deleted file mode 100644 index 0d19eb827..000000000 --- a/docker/rocm/migraphx/targets/cpu/layernorm.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_layernorm : dnnl_op -{ - float epsilon = 1e-12f; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.epsilon, "epsilon")); - } - - std::string name() const { return "dnnl::layernorm"; } - - shape compute_shape(std::vector inputs) const - { - // Compensate for allocation - inputs.pop_back(); - check_shapes{this->trim_post_op_inputs(inputs), *this}.has(1); - auto s = inputs.at(0); - // Call to get_primitive to make sure an algo is available - this->get_primitive(this->to_memory_desc(s, inputs)); - return s; - } - - dnnl::layer_normalization_forward::desc - get_desc(const std::unordered_map& m) const - { - return {dnnl::prop_kind::forward_inference, - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), - 1e-12f, - dnnl::normalization_flags::none}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/logsoftmax.cpp b/docker/rocm/migraphx/targets/cpu/logsoftmax.cpp deleted file mode 100644 index e4bb88dc8..000000000 --- a/docker/rocm/migraphx/targets/cpu/logsoftmax.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_logsoftmax : dnnl_extend_op -{ - dnnl::logsoftmax_forward::desc - get_desc(const std::unordered_map& m) const - { - int axis = this->op.axis; - return {dnnl::prop_kind::forward_inference, m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC_0)), axis}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/lowering.cpp b/docker/rocm/migraphx/targets/cpu/lowering.cpp deleted file mode 100644 index a68eae820..000000000 --- a/docker/rocm/migraphx/targets/cpu/lowering.cpp +++ /dev/null @@ -1,502 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -template -T zero(const T&) -{ - return T(0); -} - -template -typename std::conditional_t{}, std::make_signed, std::enable_if>:: - type - make_signed(T x) -{ - return x; -} - -struct cpu_im2col -{ - op::im2col op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - static std::string name() { return "cpu::im2col"; } - shape compute_shape(const std::vector& inputs) const - { - return op.normalize_compute_shape(inputs); - } - - argument compute(context&, const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - auto input_shape = args[0].get_shape(); - auto weights_shape = args[1].get_shape(); - visit_all(result, args[0])([&](auto col, auto input) { - const std::size_t& height = input_shape.lens()[2]; - const std::size_t& width = input_shape.lens()[3]; - const std::size_t& channels = weights_shape.lens()[1]; - const std::size_t& kernel_h = weights_shape.lens()[2]; - const std::size_t& kernel_w = weights_shape.lens()[3]; - const std::size_t& pad_h = op.padding[0]; - const std::size_t& pad_w = op.padding[1]; - const std::size_t& stride_h = op.stride[0]; - const std::size_t& stride_w = op.stride[1]; - - long kdiv2_h = long(kernel_h) / 2; - long kdiv2_w = long(kernel_w) / 2; - // calculate output sizes - const std::size_t col_height = (height - kernel_h + 2 * pad_h) / stride_h + 1; - const std::size_t col_width = (width - kernel_w + 2 * pad_w) / stride_w + 1; - // account for padding for the starting position of the input pixels - long iinput = kdiv2_h - long(pad_h); - // loop over output pixels (ioutput, joutput) - for(std::size_t ioutput = 0; ioutput < col_height; ioutput++, iinput += stride_h) - { - long jinput = kdiv2_w - long(pad_w); - for(std::size_t joutput = 0; joutput < col_width; joutput++, jinput += stride_w) - { - // compute linear index for output - std::size_t ldx = ioutput * col_width + joutput; - std::size_t p = 0; - dfor(channels, - kernel_h, - kernel_w)([&](std::size_t c, std::size_t koffset, std::size_t loffset) { - auto idx = iinput + long(koffset) - kdiv2_h; - auto jdx = jinput + long(loffset) - kdiv2_w; - col(ldx, p) = - ((idx >= 0) and (idx < height) and (jdx >= 0) and (jdx < width)) - ? input(0, c, idx, jdx) - : 0; - p++; - }); - } - } - }); - return result; - } -}; -MIGRAPHX_REGISTER_OP(cpu_im2col) - -struct cpu_op -{ - operation op = op::identity{}; - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - std::string name() const { return "cpu::op"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - argument compute(context&, const shape& output_shape, const std::vector& args) const - { - return op.compute(output_shape, args); - } - value to_value() const - { - value v; - v["name"] = op.name(); - v["operator"] = op.to_value(); - return v; - } - void from_value(const value& v) - { - op = make_op(v.at("name").to(), v.at("operator")); - } - friend std::ostream& operator<<(std::ostream& os, const cpu_op& x) - { - os << "cpu::" << x.op; - return os; - } -}; -MIGRAPHX_REGISTER_OP(cpu_op) - -struct cpu_pad -{ - op::pad op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "cpu::pad"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - argument compute(context&, const shape& output_shape, std::vector args) const - { - assert(output_shape.standard()); - argument result{output_shape}; - result.visit([&](auto output) { - using type = typename decltype(output)::value_type; - std::fill(output.begin(), output.end(), pad_clamp(op.value)); - }); - - visit_all(result, args[0])([&](auto output, auto input) { - shape_for_each(input.get_shape(), [&](const auto& idx) { - std::vector new_idx(idx.size()); - std::transform( - idx.begin(), idx.end(), op.pads.begin(), new_idx.begin(), [](auto i, auto j) { - return i + j; - }); - output(new_idx.begin(), new_idx.end()) = input(idx.begin(), idx.end()); - }); - }); - - return result; - } -}; -MIGRAPHX_REGISTER_OP(cpu_pad) - -struct cpu_rnn_var_sl_last_output -{ - op::rnn_var_sl_last_output op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "cpu::rnn_var_sl_last_output"; } - - shape compute_shape(std::vector inputs) const - { - return op.compute_shape(std::move(inputs)); - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - auto out_comp_lens = args[0].get_shape().lens(); - out_comp_lens[0] = 1; - shape out_comp_s{output_shape.type(), out_comp_lens}; - - visit_all(result, args[0])([&](auto output, auto input) { - args[1].visit([&](auto seq_lens) { - par_for(output_shape.elements(), [&](auto i) { - auto idx = out_comp_s.multi(i); - auto b = idx[2]; - if(op.direction == op::rnn_direction::reverse or idx[1] == 1) - { - idx[0] = 0; - } - else - { - idx[0] = seq_lens[b] - 1; - } - output[i] = input(idx.begin(), idx.end()); - }); - }); - }); - - return result; - } -}; -MIGRAPHX_REGISTER_OP(cpu_rnn_var_sl_last_output) - -struct cpu_apply -{ - module* modl; - std::unordered_map> apply_map{}; - instruction_ref last{}; - - void extend_op(const std::string& op_name, const std::string& cpu_name, bool allocate = true) - { - apply_map.emplace(op_name, [=](instruction_ref ins) { - auto&& op = ins->get_operator(); - if(allocate) - return replace(ins, make_op(cpu_name, op.to_value())); - return modl->replace_instruction(ins, make_op(cpu_name, op.to_value()), ins->inputs()); - }); - } - - void extend_dnnl_algos(const std::string& dnnl_name, - const std::vector>& algos) - { - for(auto&& pp : algos) - { - std::string op_name = pp.first; - std::string algo = pp.second; - apply_map.emplace(op_name, [=](instruction_ref ins) { - auto v = ins->get_operator().to_value(); - if(not v.is_object()) - return ins; - v["algo"] = algo; - auto op = make_op(dnnl_name, v); - return replace(ins, op); - }); - } - } - - template - auto fuse_match(M matcher, const operation& op, const std::vector& bind_inputs) - { - return match::make_match_finder(matcher, [=](auto&, const auto& r) { - auto ins = r.result; - std::vector inputs; - std::transform(bind_inputs.begin(), - bind_inputs.end(), - std::back_inserter(inputs), - [&](const auto& s) { return r.instructions[s]; }); - inputs.push_back(this->insert_allocation(ins, ins->get_shape())); - modl->replace_instruction(ins, op, inputs); - }); - } - - void init() - { - extend_dnnl_algos("dnnl::binary", - { - {"add", "binary_add"}, - {"div", "binary_div"}, - {"max", "binary_max"}, - {"min", "binary_min"}, - {"mul", "binary_mul"}, - }); - - extend_dnnl_algos("dnnl::eltwise", - { - {"abs", "eltwise_abs"}, - {"elu", "eltwise_elu"}, - {"exp", "eltwise_exp"}, - {"log", "eltwise_log"}, - {"relu", "eltwise_relu"}, - {"sqrt", "eltwise_sqrt"}, - {"tanh", "eltwise_tanh"}, - }); - - extend_dnnl_algos("dnnl::reduction", - { - {"reduce_max", "reduction_max"}, - {"reduce_mean", "reduction_mean"}, - {"reduce_min", "reduction_min"}, - {"reduce_sum", "reduction_sum"}, - }); - extend_op("concat", "dnnl::concat"); - extend_op("contiguous", "dnnl::reorder"); - extend_op("convolution", "dnnl::convolution"); -#ifndef MIGRAPHX_ENABLE_ZENDNN - extend_op("convolution_backwards", "dnnl::convolution_backwards"); - extend_op("dot", "dnnl::dot"); -#endif - extend_op("erf", "cpu::erf"); - extend_op("gather", "cpu::gather"); - extend_op("logsoftmax", "dnnl::logsoftmax"); - extend_op("lrn", "dnnl::lrn"); - extend_op("softmax", "dnnl::softmax"); - - extend_op("im2col", "cpu::im2col", false); - extend_op("leaky_relu", "cpu::leaky_relu", false); - extend_op("pad", "cpu::pad", false); - extend_op("rnn_var_sl_last_output", "cpu::rnn_var_sl_last_output", false); - } - - void apply() - { - init(); - // Apply fusion matchers first - match::find_matches(*modl, - fuse_match(match::gelu_erf(), - make_op("dnnl::eltwise", {{"algo", "eltwise_gelu_erf"}}), - {"x"}), - fuse_match(match::gelu_tanh(), - make_op("dnnl::eltwise", {{"algo", "eltwise_gelu_tanh"}}), - {"x"}), - fuse_match(match::layernorm(), make_op("dnnl::layernorm"), {"x"})); - // Apply these operators first so the inputs can be const folded - for(auto it : iterator_for(*modl)) - { - // skip lowering if input has fp8 as one of the inputs since oneDNN doesn't have fp8 - // supported yet. - if(std::any_of(it->inputs().begin(), it->inputs().end(), [](const auto& i) { - return contains(fp8_types{}.get(), i->get_shape().type()); - })) - continue; - if(it->name() == "pow") - { - apply_pow(it); - } - } - for(auto it : iterator_for(*modl)) - { - // skip lowering if input has fp8 as one of the inputs since oneDNN doesn't have fp8 - // supported yet. - if(std::any_of(it->inputs().begin(), it->inputs().end(), [](const auto& i) { - return contains(fp8_types{}.get(), i->get_shape().type()); - })) - continue; - if(it->name() == "pooling") - { - apply_pooling(it); - } - else if(it->name() == "reshape") - { - apply_reshape(it); - } - else if(apply_map.count(it->name()) > 0) - { - apply_map.at(it->name())(it); - } - } - } - - instruction_ref apply_pow(instruction_ref ins) const - { - auto beta = read_scalar(ins->inputs()[1]); - if(beta.empty()) - return ins; - return replace(ins, - make_op("dnnl::eltwise", - {{"algo", "eltwise_pow"}, {"alpha", 1.0}, {"beta", beta.front()}}), - {ins->inputs().front()}); - } - - // TODO: update lowering to run the reference - // code when OneDNN can't execute pooling for a CPU - - // OneDNN has a limitation on padding size for pooling. see - // https://oneapi-src.github.io/oneDNN/dev_guide_convolution.html#doxid-dev-guide-convolution - - // padding = {2}; stride = {1}; lengths = {3} succeeds in oneDNN but - // padding = {2}; stride = {1}; lengths = {2} fails. - // Also, the referenced documentation contains a max. dimension size of 14 for the kernel - // ("weights tensor") that MIGraphX doesn't enforce. - instruction_ref apply_pooling(instruction_ref ins) const - { - auto&& op = ins->get_operator(); - auto v = op.to_value(); - if(has_op("dnnl::pooling") and ins->get_shape().type() == shape::type_t::float_type and - not v["ceil_mode"].to() and - v["mode"].to() != op::pooling_mode::lpnorm) - return replace(ins, make_op("dnnl::pooling", op.to_value())); - return ins; - } - /* - Lowers reshape copy operator to reshape lazy by inserting contiguous operators around it. - Contiguous ops will later by removed by eliminate_contiguous pass. - */ - instruction_ref apply_reshape(instruction_ref ins) const - { - std::vector before_contiguous_args = ins->inputs(); - auto before_alloc = - insert_allocation(ins, before_contiguous_args.front()->get_shape().as_standard()); - before_contiguous_args.push_back(before_alloc); - auto before_contig = - modl->insert_instruction(ins, make_op("dnnl::reorder"), {before_contiguous_args}); - - auto new_lazy_reshape = modl->insert_instruction( - ins, - make_op("reshape_lazy", {{"dims", {ins->get_operator().to_value().at("dims")}}}), - before_contig); - - std::vector after_contiguous_args = {new_lazy_reshape}; - auto after_alloc = insert_allocation(new_lazy_reshape, new_lazy_reshape->get_shape()); - after_contiguous_args.push_back(after_alloc); - return modl->replace_instruction(ins, make_op("dnnl::reorder"), after_contiguous_args); - } - - template - static std::vector read_scalar(instruction_ref ins) - { - if(ins->name() == "contiguous") - return read_scalar(ins->inputs().front()); - if(ins->get_shape().elements() != 1 and not ins->get_shape().scalar()) - return {}; - auto r = ins->eval(); - if(r.empty()) - return {}; - return {r.at()}; - } - - instruction_ref replace(instruction_ref ins, const operation& op) const - { - return replace(ins, op, ins->inputs()); - } - - instruction_ref - replace(instruction_ref ins, const operation& op, std::vector inputs) const - { - inputs.push_back(insert_allocation(ins, ins->get_shape())); - return modl->replace_instruction(ins, op, inputs); - } - - instruction_ref insert_allocation(instruction_ref ins, const shape& s) const - { - return modl->insert_instruction(ins, make_op("allocate", {{"shape", to_value(s)}})); - } -}; - -void lowering::apply(module& m) const { cpu_apply{&m}.apply(); } - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/lrn.cpp b/docker/rocm/migraphx/targets/cpu/lrn.cpp deleted file mode 100644 index bd4c27129..000000000 --- a/docker/rocm/migraphx/targets/cpu/lrn.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_lrn : dnnl_extend_op -{ - dnnl::lrn_forward::desc get_desc(const std::unordered_map& m) const - { - return {dnnl::prop_kind::forward_inference, - dnnl::algorithm::lrn_across_channels, - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC_0)), - this->op.size, - this->op.alpha, - this->op.beta, - this->op.bias}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/mod.cpp b/docker/rocm/migraphx/targets/cpu/mod.cpp deleted file mode 100644 index e28bdb19d..000000000 --- a/docker/rocm/migraphx/targets/cpu/mod.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -template struct cpu_binary; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/pooling.cpp b/docker/rocm/migraphx/targets/cpu/pooling.cpp deleted file mode 100644 index d10ed75a6..000000000 --- a/docker/rocm/migraphx/targets/cpu/pooling.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_pooling : dnnl_extend_op -{ - std::vector arg_map(int) const { return {MIGRAPHX_DNNL_PREFIX(ARG_SRC)}; } - - dnnl::algorithm get_algo() const - { - switch(op.mode) - { - case op::pooling_mode::max: return dnnl::algorithm::pooling_max; - case op::pooling_mode::average: - return op.count_include_pad ? dnnl::algorithm::pooling_avg_include_padding - : dnnl::algorithm::pooling_avg_exclude_padding; - case op::pooling_mode::lpnorm: MIGRAPHX_THROW("Lpnorn pooling mode not supported"); - } - MIGRAPHX_THROW("Unknown pooling mode"); - } - - dnnl::pooling_v2_forward::desc - get_desc(const std::unordered_map& m) const - { - auto algo = get_algo(); - auto kdims = op.kdims(); - std::vector padding_l(op.padding.begin(), op.padding.begin() + kdims); - std::vector padding_r(op.padding.begin() + kdims, op.padding.end()); - // Note: It is not documented, but the default dilation seems to be 0 instead of 1. - // We need to offset dilations with -1. - std::vector dilations; - std::transform(op.dilations.cbegin(), - op.dilations.cend(), - std::back_inserter(dilations), - [](size_t d) { return d - 1; }); - return {dnnl::prop_kind::forward_inference, - algo, - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST)), - to_dnnl_dims(op.stride), - to_dnnl_dims(op.lengths), - to_dnnl_dims(dilations), - to_dnnl_dims(padding_l), - to_dnnl_dims(padding_r)}; - } -}; - -} // namespace cpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/preallocate.cpp b/docker/rocm/migraphx/targets/cpu/preallocate.cpp deleted file mode 100644 index d831a1942..000000000 --- a/docker/rocm/migraphx/targets/cpu/preallocate.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct cpu_preallocate : auto_register_op -{ - shape s; - std::string id = ""; - argument data; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape"), f(self.id, "id")); - } - - std::string name() const { return "cpu::preallocate"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return s; - } - argument compute(context&, const shape&, const std::vector&) const { return data; } - void finalize(context&, const shape&, const std::vector&) { data = argument(s); } - lifetime get_lifetime() const { return lifetime::global; } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/reduction.cpp b/docker/rocm/migraphx/targets/cpu/reduction.cpp deleted file mode 100644 index e0a7517ee..000000000 --- a/docker/rocm/migraphx/targets/cpu/reduction.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_reduction : dnnl_op -{ - std::string algo; - std::vector axes{}; - template - static auto reflect(Self& self, F f) - { - return pack_join(self.reflect_base(self, f), - pack(f(self.algo, "algo"), f(self.axes, "axes"))); - } - - std::string name() const { return "dnnl::reduction"; } - - shape compute_shape(std::vector inputs) const - { - // Compensate for allocation - inputs.pop_back(); - check_shapes{this->trim_post_op_inputs(inputs), *this}.has(1).standard(); - auto s = inputs.at(0); - auto lens = s.lens(); - for(auto axis : axes) - { - lens[axis] = 1; - } - auto r = shape{s.type(), lens}; - // Call to get_primitive to make sure an algo is available - this->get_primitive(this->to_memory_desc(r, inputs)); - return r; - } - - dnnl::reduction::desc get_desc(const std::unordered_map& m) const - { - return {to_dnnl_algo(algo), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), - m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST)), - 0, - 0}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/reorder.cpp b/docker/rocm/migraphx/targets/cpu/reorder.cpp deleted file mode 100644 index c549a6013..000000000 --- a/docker/rocm/migraphx/targets/cpu/reorder.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_reorder : dnnl_op -{ - std::string name() const { return "dnnl::reorder"; } - - shape adjust_shape(const shape& x, int, const shape&) const { return x; } - - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(2); - auto r = inputs.back(); - // Call to get_primitive to make sure an algo is available - this->get_primitive(this->to_memory_desc(r, inputs)); - return r; - } - // Custom desc class since its missing in dnnl - struct desc - { - dnnl::memory::desc src; - dnnl::memory::desc dst; - }; - desc get_desc(const std::unordered_map& m) const - { - return {m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC)), m.at(MIGRAPHX_DNNL_PREFIX(ARG_DST))}; - } - - auto get_primitive_desc(const desc& d, const dnnl::primitive_attr& attr) const - { - auto& engine = get_dnnl_context().engine; - return dnnl::reorder::primitive_desc(engine, d.src, engine, d.dst, attr); - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/softmax.cpp b/docker/rocm/migraphx/targets/cpu/softmax.cpp deleted file mode 100644 index 8c3610f23..000000000 --- a/docker/rocm/migraphx/targets/cpu/softmax.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct dnnl_softmax : dnnl_extend_op -{ - dnnl::softmax_forward::desc get_desc(const std::unordered_map& m) const - { - int axis = this->op.axis; - return {dnnl::prop_kind::forward_inference, m.at(MIGRAPHX_DNNL_PREFIX(ARG_SRC_0)), axis}; - } -}; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/sub.cpp b/docker/rocm/migraphx/targets/cpu/sub.cpp deleted file mode 100644 index 8f3436071..000000000 --- a/docker/rocm/migraphx/targets/cpu/sub.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -template struct cpu_binary; - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/target.cpp b/docker/rocm/migraphx/targets/cpu/target.cpp deleted file mode 100644 index e148aa5b6..000000000 --- a/docker/rocm/migraphx/targets/cpu/target.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -std::string target::name() const { return "cpu"; } - -// cppcheck-suppress constParameterReference -std::vector target::get_passes(migraphx::context& gctx, const compile_options&) const -{ - auto& ctx = any_cast(gctx); - std::set unsupported_types(shape::types().begin(), shape::types().end()); - std::set unsupported_ops{ - "all", "scatternd_add", "scatternd_mul", "scatternd_none"}; - unsupported_types.erase(shape::type_t::float_type); - return {normalize_ops{}, - rewrite_quantization{}, - dead_code_elimination{}, - eliminate_data_type{unsupported_types, shape::type_t::float_type, unsupported_ops}, - dead_code_elimination{}, - simplify_reshapes{}, - eliminate_convert{}, - eliminate_identity{}, - eliminate_pad{}, - dead_code_elimination{}, - rewrite_rnn{}, - dead_code_elimination{}, - eliminate_common_subexpression{}, - dead_code_elimination{}, - simplify_algebra{}, - simplify_reshapes{}, - eliminate_convert{}, - dead_code_elimination{}, - simplify_reshapes{}, - eliminate_convert{}, - dead_code_elimination{}, - simplify_algebra{}, - simplify_reshapes{}, - eliminate_convert{}, - dead_code_elimination{}, - propagate_constant{}, - dead_code_elimination{}, - auto_contiguous{}, - lowering{}, - eliminate_contiguous{"dnnl::reorder"}, - dead_code_elimination{}, - replace_allocate{cpu_allocation_model{}}, - dead_code_elimination{}, - adjust_allocation{cpu_allocation_model{}}, - dead_code_elimination{}, - fuse_ops{&ctx}, - dead_code_elimination{}, - write_literals{}, - dead_code_elimination{}, - memory_coloring{"cpu::allocate"}, - dead_code_elimination{}, - preallocate_param{"scratch", cpu_allocation_model{}}, - dead_code_elimination{}}; -} - -argument target::allocate(const shape& s) const { return fill_argument(s, 0); } - -MIGRAPHX_REGISTER_TARGET(target); - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/cpu/write_literals.cpp b/docker/rocm/migraphx/targets/cpu/write_literals.cpp deleted file mode 100644 index 0899df4e8..000000000 --- a/docker/rocm/migraphx/targets/cpu/write_literals.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace cpu { - -struct cpu_literal -{ - argument data; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.data, "data")); - } - - std::string name() const { return "cpu::literal"; } - - shape compute_shape(const std::vector&) const { return data.get_shape(); } - - argument compute(const shape&, const std::vector&) const { return data; } - - friend std::ostream& operator<<(std::ostream& os, const cpu_literal& x) - { - os << x.name(); - return os; - } -}; -MIGRAPHX_REGISTER_OP(cpu_literal); - -void write_literals::apply(module& m) const -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "@literal") - continue; - m.replace_instruction(ins, cpu_literal{ins->get_literal().get_argument()}); - } -} - -} // namespace cpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/fpga/CMakeLists.txt b/docker/rocm/migraphx/targets/fpga/CMakeLists.txt deleted file mode 100644 index 11b47b9b2..000000000 --- a/docker/rocm/migraphx/targets/fpga/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -add_library(migraphx_fpga - target.cpp - lowering.cpp - subgraph.cpp - vitis_ai_adapter.cpp -) - -set_target_properties(migraphx_fpga PROPERTIES EXPORT_NAME fpga) -rocm_set_soversion(migraphx_fpga ${MIGRAPHX_SO_VERSION}) - -rocm_clang_tidy_check(migraphx_fpga) -target_link_libraries(migraphx_fpga migraphx) - -rocm_install_targets( - PRIVATE - TARGETS migraphx_fpga - INCLUDE - ${CMAKE_CURRENT_SOURCE_DIR}/include -) diff --git a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/context.hpp b/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/context.hpp deleted file mode 100644 index 2c8242a76..000000000 --- a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/context.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_FPGA_CONTEXT_HPP -#define MIGRAPHX_GUARD_FPGA_CONTEXT_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace fpga { - -struct context -{ - int id = 0; - - void finish() const {} -}; - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_FPGA_CONTEXT_HPP diff --git a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/lowering.hpp b/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/lowering.hpp deleted file mode 100644 index dc8a7bc6b..000000000 --- a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/lowering.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_FPGA_LOWERING_HPP -#define MIGRAPHX_GUARD_FPGA_LOWERING_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace fpga { - -struct lowering -{ - context* ctx = nullptr; - std::string name() const { return "fpga::lowering"; } - void apply(module& m) const; -}; - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_FPGA_LOWERING_HPP diff --git a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/subgraph.hpp b/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/subgraph.hpp deleted file mode 100644 index 62f68b09d..000000000 --- a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/subgraph.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_FPGA_SUBGRAPH_HPP -#define MIGRAPHX_GUARD_FPGA_SUBGRAPH_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace fpga { - -struct subgraph -{ - std::string name() const { return "fpga::subgraph"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_FPGA_SUBGRAPH_HPP diff --git a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/target.hpp b/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/target.hpp deleted file mode 100644 index dbcb0bcff..000000000 --- a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/target.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_FPGA_TARGET_HPP -#define MIGRAPHX_GUARD_FPGA_TARGET_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct pass; -namespace fpga { - -struct target -{ - std::string name() const; - std::vector get_passes(migraphx::context& ctx, const compile_options&) const; - migraphx::context get_context() const { return context{}; } - supported_segments find_supported(const_module_ref mod, support_metric m) const; - argument copy_to(const argument& arg) const { return arg; } - argument copy_from(const argument& arg) const { return arg; } - argument allocate(const shape& s) const; -}; - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_FPGA_TARGET_HPP diff --git a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/vitis_ai_adapter.hpp b/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/vitis_ai_adapter.hpp deleted file mode 100644 index 64d2300c4..000000000 --- a/docker/rocm/migraphx/targets/fpga/include/migraphx/fpga/vitis_ai_adapter.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_FPGA_VITIS_AI_ADAPTER_HPP -#define MIGRAPHX_GUARD_FPGA_VITIS_AI_ADAPTER_HPP - -#include - -#include -#include - -namespace vitis_ai { - -class x_model -{ - migraphx::shape shape; - - public: - migraphx::shape get_shape() const; - void set_shape(migraphx::shape); -}; - -x_model create_xmodel(migraphx::const_module_ref mod); - -migraphx::argument execute(const x_model& xmodel, - const migraphx::shape& output_shape, - std::vector& args); - -} // namespace vitis_ai - -#endif // MIGRAPHX_GUARD_FPGA_VITIS_AI_ADAPTER_HPP diff --git a/docker/rocm/migraphx/targets/fpga/lowering.cpp b/docker/rocm/migraphx/targets/fpga/lowering.cpp deleted file mode 100644 index ad17dc8d1..000000000 --- a/docker/rocm/migraphx/targets/fpga/lowering.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include - -#include "migraphx/fpga/vitis_ai_adapter.hpp" - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace fpga { - -struct fpga_vitis_op -{ - fpga_vitis_op() = default; - explicit fpga_vitis_op(vitis_ai::x_model model) : xmodel(std::move(model)){}; - - vitis_ai::x_model xmodel; - int dummy = 0; - - template - static auto reflect(Self& self, F f) - { - // return pack(f(self.xmodel, "xmodel")); - return pack(f(self.dummy, "dummy")); - } - - std::string name() const { return "fpga::vitis_ai"; } - - shape compute_shape(const std::vector& inputs) const - { - (void)inputs; - return xmodel.get_shape(); - } - - argument - compute(const context& ctx, const shape& output_shape, std::vector args) const - { - std::cout << "The context is " << ctx.id << std::endl; - return ::vitis_ai::execute(xmodel, output_shape, args); - } -}; -MIGRAPHX_REGISTER_OP(fpga_vitis_op) - -void lowering::apply(module& m) const -{ - auto* mod = &m; - - // test modifying the context from a pass - ctx->id = 2; - - for(auto it : iterator_for(*mod)) - { - if(it->name() == "fpga::vitis_placeholder") - { - assert(it->module_inputs().size() == 1); - auto xmodel = ::vitis_ai::create_xmodel(it->module_inputs()[0]); - mod->replace_instruction(it, fpga_vitis_op{xmodel}, it->inputs()); - } - } -} - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/fpga/subgraph.cpp b/docker/rocm/migraphx/targets/fpga/subgraph.cpp deleted file mode 100644 index d0e09a5de..000000000 --- a/docker/rocm/migraphx/targets/fpga/subgraph.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -#include -#include "migraphx/iterator.hpp" -#include -#include "migraphx/make_op.hpp" -#include "migraphx/module.hpp" -#include "migraphx/ranges.hpp" -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace fpga { - -struct fpga_placeholder_op -{ - fpga_placeholder_op() = default; - - int dummy = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.dummy, "dummy")); - } - - std::string name() const { return "fpga::vitis_placeholder"; } - - shape compute_shape(const std::vector& inputs, std::vector mods) const - { - (void)inputs; - if(mods.size() != 1) - { - MIGRAPHX_THROW("should have one submodule."); - } - module_ref sm = mods.front(); - if(sm->get_output_shapes().size() != 1) - MIGRAPHX_THROW("Only one return"); - return sm->get_output_shapes().front(); - } -}; -MIGRAPHX_REGISTER_OP(fpga_placeholder_op) - -bool is_fpga_instr(migraphx::instruction_ref it) -{ - // assuming all instructions that aren't @param, @literal, or input data are fpga instrs - if(migraphx::starts_with(it->name(), "@")) - { - return false; - } - // no inputs to the instr means it's input data - if(it->inputs().empty()) - { - return false; - } - return true; -} - -void subgraph::apply(module_pass_manager& mpm) const -{ - auto& mod = mpm.get_module(); - auto* pm = mpm.create_module(mod.name() + ":fpga"); - pm->set_bypass(); - - migraphx::instruction_ref first = mod.end(); - migraphx::instruction_ref last; - std::vector literal_inputs; - for(auto it : iterator_for(mod)) - { - // assuming we want all the params/literals as inputs to the FPGA submodule - if(migraphx::starts_with(it->name(), "@param") or - migraphx::starts_with(it->name(), "@literal")) - { - literal_inputs.push_back(it); - } - if(is_fpga_instr(it)) - { - if(first == mod.end()) - { - first = it; - } - last = it; - } - } - - // TODO(varunsh): this code may be replaceable by code in the fuse_pointwise pass - - // assuming all FPGA instructions are in one contiguous range - pm->insert_instructions(pm->end(), first, std::next(last), {}); - migraphx::instruction_ref placeholder_ins; - for(auto it : iterator_for(mod)) - { - if(migraphx::starts_with(it->name(), "@return")) - { - placeholder_ins = mod.insert_instruction( - it, migraphx::make_op("fpga::vitis_placeholder"), literal_inputs, {pm}); - break; - } - } - - mod.replace_return({placeholder_ins}); -} - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/fpga/target.cpp b/docker/rocm/migraphx/targets/fpga/target.cpp deleted file mode 100644 index 570779fff..000000000 --- a/docker/rocm/migraphx/targets/fpga/target.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace fpga { - -std::string target::name() const { return "fpga"; } - -std::vector target::get_passes(migraphx::context& gctx, const compile_options&) const -{ - // not sure if all these passes are needed but they were copied from ref/ - auto& ctx = any_cast(gctx); - return {normalize_ops{}, - eliminate_pad{}, - dead_code_elimination{}, - insert_pad{}, - dead_code_elimination{}, - rewrite_rnn{}, - dead_code_elimination{}, - auto_contiguous{}, - dead_code_elimination{}, - subgraph{}, - dead_code_elimination{}, - lowering{&ctx}, - dead_code_elimination{}}; -} - -argument target::allocate(const shape& s) const { return fill_argument(s, 0); } - -supported_segments target::find_supported(const_module_ref mod, support_metric m) const -{ - (void)m; - - supported_segment instrs; - for(const auto ins : iterator_for(*mod)) - { - instrs.instructions.insert(ins); - } - instrs.metric = 1; // arbitrary value - return {instrs}; -} - -MIGRAPHX_REGISTER_TARGET(target); - -} // namespace fpga -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/fpga/vitis_ai_adapter.cpp b/docker/rocm/migraphx/targets/fpga/vitis_ai_adapter.cpp deleted file mode 100644 index fa4ecdc68..000000000 --- a/docker/rocm/migraphx/targets/fpga/vitis_ai_adapter.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "migraphx/fpga/vitis_ai_adapter.hpp" - -#include "migraphx/module.hpp" - -#include "migraphx/stringutils.hpp" -namespace vitis_ai { - -migraphx::shape x_model::get_shape() const { return shape; }; - -void x_model::set_shape(migraphx::shape s) { shape = s; } - -x_model create_xmodel(migraphx::const_module_ref mod) -{ - std::cout << "Calling an external function: create_xmodel!\n"; - x_model xmodel; - xmodel.set_shape(migraphx::shape(mod->get_output_shapes())); - return xmodel; -} - -migraphx::argument execute(const x_model& xmodel, - const migraphx::shape& output_shape, - std::vector& args) -{ - (void)xmodel; - - std::cout << "Calling an external function: execute!\n"; - - std::cout << "Output Shape: " << output_shape << std::endl; - std::cout << "Args: " << args.size() << std::endl; - for(const auto& arg : args) - { - std::cout << " " << arg.get_shape() << std::endl; - } - std::cout << std::endl; - - migraphx::argument result{output_shape}; - - return result; -} - -} // namespace vitis_ai diff --git a/docker/rocm/migraphx/targets/gpu/CMakeLists.txt b/docker/rocm/migraphx/targets/gpu/CMakeLists.txt deleted file mode 100644 index 82cc1fb0a..000000000 --- a/docker/rocm/migraphx/targets/gpu/CMakeLists.txt +++ /dev/null @@ -1,407 +0,0 @@ -# #################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# #################################################################################### - -find_package(hip REQUIRED) -if(NOT GPU_TARGETS) - set(fatal_msg "HIP package is broken and has no GPU_TARGETS. Please pass GPU_TARGETS to cmake.") - if(NOT WIN32) - set(fatal_msg "${fatal_msg}\nUse -DGPU_TARGETS=$(/opt/rocm/bin/rocminfo | grep -o -m1 'gfx.*') to build for your GPU.") - endif() - message(FATAL_ERROR ${fatal_msg}) -endif() - -if(MIGRAPHX_USE_MIOPEN) - find_package(miopen REQUIRED) - message(STATUS "MIGraphX is using MIOpen") -else() - message(STATUS "MIGraphX is not using MIOpen") -endif() - -if(MIGRAPHX_USE_ROCBLAS) - # rocblas - find_package(rocblas REQUIRED) - message(STATUS "MIGraphX build with rocBLAS") -else() - message(STATUS "MIGraphX build without rocBLAS") -endif() - -if(MIGRAPHX_USE_HIPBLASLT) - # hipblaslt - find_package(hipblaslt REQUIRED) - # Making hipblas required to workaround the broken hipblaslt package. - find_package(hipblas REQUIRED) - message(STATUS "MIGraphx build with hipBLAS and hipBLASLt") -else() - message(STATUS "MIGraphX build without hipBLAS and hipBLASLt") -endif() - -if(MIGRAPHX_USE_COMPOSABLEKERNEL) - find_package(composable_kernel 1.0.0 REQUIRED COMPONENTS jit_library) -endif() - -if(BUILD_DEV) - set(MIGRAPHX_USE_HIPRTC OFF CACHE BOOL "Use hipRTC APIs") -else() - set(MIGRAPHX_USE_HIPRTC ON CACHE BOOL "Use hipRTC APIs") -endif() - -file(GLOB KERNEL_FILES CONFIGURE_DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/kernels/include/migraphx/kernels/*.hpp) - -if(NOT MIGRAPHX_USE_COMPOSABLEKERNEL) - list(REMOVE_ITEM KERNEL_FILES - ${CMAKE_CURRENT_SOURCE_DIR}/kernels/include/migraphx/kernels/ck_gemm.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/kernels/include/migraphx/kernels/ck_gemm_softmax_gemm.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/kernels/include/migraphx/kernels/ck.hpp) -endif() - -include(Embed) -add_embed_library(migraphx_kernels ${KERNEL_FILES} RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/kernels/include/) - -configure_file(device/targets.hpp.in include/migraphx/gpu/device/targets.hpp) -file(GLOB DEVICE_GPU_SRCS CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/device/*.cpp) -add_library(migraphx_device ${DEVICE_GPU_SRCS}) - -add_library(compile_for_gpu INTERFACE) -target_compile_features(compile_for_gpu INTERFACE cxx_std_17) -target_compile_options(compile_for_gpu INTERFACE -fno-gpu-rdc -Wno-cuda-compat -Wno-unused-command-line-argument -Xclang -fnative-half-arguments-and-returns) -target_link_options(compile_for_gpu INTERFACE -fno-gpu-rdc -Wno-invalid-command-line-argument -Wno-unused-command-line-argument -Wno-option-ignored) -target_link_libraries(compile_for_gpu INTERFACE hip::device) -check_cxx_compiler_flag("--cuda-host-only -fhip-lambda-host-device -x hip" HAS_HIP_LAMBDA_HOST_DEVICE) - -if(HAS_HIP_LAMBDA_HOST_DEVICE) - message(STATUS "Enable -fhip-lambda-host-device") - target_compile_options(compile_for_gpu INTERFACE -fhip-lambda-host-device) -endif() - -set_target_properties(migraphx_device PROPERTIES EXPORT_NAME device) -rocm_set_soversion(migraphx_device ${MIGRAPHX_SO_VERSION}) -rocm_clang_tidy_check(migraphx_device) -target_link_libraries(migraphx_device PUBLIC migraphx) -target_link_libraries(migraphx_device PRIVATE compile_for_gpu) -if(NOT MIGRAPHX_USE_MIOPEN AND NOT MIGRAPHX_USE_ROCBLAS) - target_link_libraries(migraphx_device INTERFACE hip::host) -endif() -target_include_directories(migraphx_device PUBLIC $) -target_include_directories(migraphx_device PRIVATE $) -target_include_directories(migraphx_device PRIVATE $) -target_compile_options(migraphx_device PRIVATE -Wno-ignored-attributes) -migraphx_generate_export_header(migraphx_device DIRECTORY migraphx/gpu/device) - -add_library(kernel_file_check EXCLUDE_FROM_ALL) - -foreach(KERNEL_FILE ${KERNEL_FILES}) - get_filename_component(KERNEL_BASE_FILE ${KERNEL_FILE} NAME_WE) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/kernels/include/migraphx/kernels/${KERNEL_BASE_FILE}.cpp "#include \n") - target_sources(kernel_file_check PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/kernels/include/migraphx/kernels/${KERNEL_BASE_FILE}.cpp) -endforeach() - -target_compile_definitions(kernel_file_check PRIVATE -DMIGRAPHX_NLOCAL=256) -target_compile_definitions(kernel_file_check PRIVATE -DMIGRAPHX_WAVEFRONTSIZE=64) -target_include_directories(kernel_file_check PRIVATE $) -target_link_libraries(kernel_file_check compile_for_gpu) -if(MIGRAPHX_USE_COMPOSABLEKERNEL) - target_link_libraries(kernel_file_check composable_kernel::jit_library) -endif() - -rocm_clang_tidy_check(kernel_file_check) - -file(GLOB JIT_GPU_SRCS CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/jit/*.cpp) - -if(NOT MIGRAPHX_USE_COMPOSABLEKERNEL) - list(REMOVE_ITEM JIT_GPU_SRCS - ${CMAKE_CURRENT_SOURCE_DIR}/jit/ck_gemm.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/jit/ck_gemm_softmax_gemm.cpp) -endif() - -if(MIGRAPHX_USE_MIOPEN) - set(MIOPEN_SRCS abs.cpp) -endif() - -add_library(migraphx_gpu - analyze_streams.cpp - allocation_model.cpp - argmax.cpp - argmin.cpp - code_object_op.cpp - compile_ops.cpp - compile_gen.cpp - compile_hip.cpp - compile_hip_code_object.cpp - compile_hipblaslt.cpp - compile_miopen.cpp - compile_pointwise.cpp - compiler.cpp - device_name.cpp - fuse_ck.cpp - fuse_mlir.cpp - fuse_ops.cpp - gemm_impl.cpp - hip.cpp - hipblaslt.cpp - hip_gemm_impl.cpp - kernel.cpp - lowering.cpp - logsoftmax.cpp - loop.cpp - lrn.cpp - mlir.cpp - multinomial.cpp - no_device.cpp - nonzero.cpp - pack_args.cpp - prefuse_ops.cpp - prepare_reduce.cpp - perfdb.cpp - pooling.cpp - problem_cache.cpp - reverse.cpp - rnn_variable_seq_lens.cpp - rocblas.cpp - schedule_model.cpp - sync_device.cpp - target.cpp - time_op.cpp - topk.cpp - write_literals.cpp - ${JIT_GPU_SRCS} - ${MIOPEN_SRCS} -) - -set_target_properties(migraphx_gpu PROPERTIES EXPORT_NAME gpu) -migraphx_generate_export_header(migraphx_gpu) - -function(register_migraphx_gpu_ops PREFIX) - foreach(OP ${ARGN}) - register_op(migraphx_gpu HEADER migraphx/gpu/${OP}.hpp OPERATORS gpu::${PREFIX}${OP} INCLUDES migraphx/gpu/context.hpp) - endforeach() -endfunction() - -register_migraphx_gpu_ops(hip_ - argmax - argmin - logsoftmax - loop - multinomial - nonzero - prefix_scan_sum - reverse - topk -) -if (MIGRAPHX_USE_MIOPEN) -register_migraphx_gpu_ops(miopen_ - abs - contiguous - lrn - pooling -) -else() -register_migraphx_gpu_ops(miopen_ - contiguous -) -endif() -register_op(migraphx_gpu - HEADER migraphx/gpu/rnn_variable_seq_lens.hpp - OPERATORS gpu::hip_rnn_var_sl_shift_sequence gpu::hip_rnn_var_sl_shift_output gpu::hip_rnn_var_sl_last_output - INCLUDES migraphx/gpu/context.hpp) -if(MIGRAPHX_USE_ROCBLAS) - register_op(migraphx_gpu - HEADER migraphx/gpu/gemm.hpp - OPERATORS gpu::rocblas_gemm gpu::rocblas_gemm - INCLUDES migraphx/gpu/context.hpp) -endif() -if(MIGRAPHX_USE_HIPBLASLT) - register_op(migraphx_gpu - HEADER migraphx/gpu/hip_gemm.hpp - OPERATORS gpu::hip_gemm gpu::hip_gemm - INCLUDES migraphx/gpu/context.hpp) -endif() -if (MIGRAPHX_USE_MIOPEN) - register_op(migraphx_gpu HEADER migraphx/gpu/convolution.hpp - OPERATORS gpu::miopen_convolution gpu::miopen_convolution gpu::miopen_convolution - INCLUDES migraphx/gpu/context.hpp) -endif() -rocm_set_soversion(migraphx_gpu ${MIGRAPHX_SO_VERSION}) -rocm_clang_tidy_check(migraphx_gpu) - -set(MIGRAPHX_ENABLE_MLIR ON CACHE BOOL "") - -if(MIGRAPHX_ENABLE_MLIR) - # Find package rocMLIR - find_package(rocMLIR 1.0.0 CONFIG REQUIRED) - message(STATUS "Build with rocMLIR::rockCompiler ${rocMLIR_VERSION}") - target_compile_definitions(migraphx_gpu PRIVATE "-DMIGRAPHX_MLIR") - # Make this private to avoid multiple inclusions of LLVM symbols. - # TODO: Fix rocMLIR's library to hide LLVM internals. - target_link_libraries(migraphx_gpu PRIVATE rocMLIR::rockCompiler) -endif() - -if(MIGRAPHX_USE_HIPRTC) - find_package(hiprtc REQUIRED) - message(STATUS "MIGraphX is using hipRTC") - target_compile_definitions(migraphx_gpu PRIVATE -DMIGRAPHX_USE_HIPRTC=1) - target_link_libraries(migraphx_gpu PUBLIC hiprtc::hiprtc) -else() - message(STATUS "MIGraphX is using HIP Clang") - - # Get flags needed to compile hip - include(TargetFlags) - target_flags(HIP_COMPILER_FLAGS hip::device) - - # Remove cuda arch flags - string(REGEX REPLACE "--cuda-gpu-arch=[a-z0-9]+ ?" "" HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") - string(REGEX REPLACE "--offload-arch=[a-z0-9:+-]+ ?" "" HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") - - # Skip library paths since hip will incorrectly treat it as a source file - string(APPEND HIP_COMPILER_FLAGS " ") - - if(WIN32) - string(REPLACE "\\" "/" HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") - endif() - foreach(_unused RANGE 2) - string(REGEX REPLACE " /[^ ]+\\.(a|so) " " " HIP_COMPILER_FLAGS "${HIP_COMPILER_FLAGS}") - endforeach() - - message(STATUS "Hip compiler flags: \"${HIP_COMPILER_FLAGS}\"") - target_compile_definitions(migraphx_gpu PRIVATE - -DMIGRAPHX_HIP_COMPILER="${CMAKE_CXX_COMPILER}" - -DMIGRAPHX_HIP_COMPILER_FLAGS="${HIP_COMPILER_FLAGS}" - ) - - if(DEFINED CMAKE_CXX_COMPILER_LAUNCHER) - if(WIN32) - execute_process(COMMAND where ${CMAKE_CXX_COMPILER_LAUNCHER} OUTPUT_VARIABLE MIGRAPHX_HIP_COMPILER_LAUNCHER) - else() - execute_process(COMMAND which ${CMAKE_CXX_COMPILER_LAUNCHER} OUTPUT_VARIABLE MIGRAPHX_HIP_COMPILER_LAUNCHER) - endif() - string(STRIP "${MIGRAPHX_HIP_COMPILER_LAUNCHER}" MIGRAPHX_HIP_COMPILER_LAUNCHER) - target_compile_definitions(migraphx_gpu PRIVATE -DMIGRAPHX_HIP_COMPILER_LAUNCHER="${MIGRAPHX_HIP_COMPILER_LAUNCHER}") - endif() -endif() - -target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_CXX_COMPILER="${CMAKE_CXX_COMPILER}") - -# Check miopen find mode api - -include(CheckLibraryExists) -if (MIGRAPHX_USE_MIOPEN) - get_target_property(MIOPEN_LOCATION MIOpen LOCATION) - target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_USE_MIOPEN=1) - check_library_exists(MIOpen "miopenHiddenSetConvolutionFindMode" "${MIOPEN_LOCATION}" HAS_FIND_MODE_API) - check_library_exists(MIOpen "miopenFindSolutions" "${MIOPEN_LOCATION}" HAS_FIND_2_API) -else() -target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_USE_MIOPEN=0) -endif() - -if(MIGRAPHX_USE_ROCBLAS) - get_target_property(ROCBLAS_LOCATION roc::rocblas LOCATION) - target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_USE_ROCBLAS=1) - # Beta API for automated GEMM tuning - check_library_exists(roc::rocblas "rocblas_gemm_ex_get_solutions" "${ROCBLAS_LOCATION}" HAS_ROCBLAS_TUNING_BETA_FEATURE_API) - # rocblas FP8 API - check_library_exists(roc::rocblas "rocblas_gemm_strided_batched_ex3" "${ROCBLAS_LOCATION}" HAS_ROCBLAS_FP8_BETA_API) -else() - target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_USE_ROCBLAS=0) -endif() - -if(MIGRAPHX_USE_HIPBLASLT) - target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_USE_HIPBLASLT=1) -else() - target_compile_definitions(migraphx_gpu PUBLIC MIGRAPHX_USE_HIPBLASLT=0) -endif() - -if(MIGRAPHX_USE_MIOPEN) - set(MIGRAPHX_USE_FIND_2_API "${HAS_FIND_2_API}" CACHE BOOL "") - - if(MIGRAPHX_USE_FIND_2_API) - check_library_exists(MIOpen "miopenSetFindOptionPreallocatedTensor" "${MIOPEN_LOCATION}" HAS_PREALLOCATION_API) - if(HAS_PREALLOCATION_API) - target_compile_definitions(migraphx_gpu PUBLIC -DMIGRAPHX_HAS_FIND_2_API -DMIGRAPHX_PREALLOCATE_MIOPEN_BUFFERS) - else() - target_compile_definitions(migraphx_gpu PUBLIC -DMIGRAPHX_HAS_FIND_2_API) - endif() - message(STATUS "MIGraphx is using Find-2.0 API of MIOpen") - else() - message(STATUS "MIGraphx is using legacy Find API in MIOpen") - endif() - - if(HAS_FIND_MODE_API) - target_compile_definitions(migraphx_gpu PUBLIC -DMIGRAPHX_HAS_FIND_MODE_API) - message(STATUS "MIGraphx is using Find Mode API of MIOpen") - else() - message(STATUS "MIOpen does not have find mode api") - endif() - - target_link_libraries(migraphx_gpu PUBLIC MIOpen) -endif() - -if(MIGRAPHX_USE_ROCBLAS) - if(HAS_ROCBLAS_TUNING_BETA_FEATURE_API) - target_compile_definitions(migraphx_gpu PUBLIC -DMIGRAPHX_USE_ROCBLAS_TUNING_API -DROCBLAS_BETA_FEATURES_API -DROCBLAS_NO_DEPRECATED_WARNINGS) - message(STATUS "MIGraphx is using Beta API of rocBLAS") - else() - message(STATUS "rocBLAS does not have User Tuning Beta API") - endif() - - if(HAS_ROCBLAS_FP8_BETA_API) - target_compile_definitions(migraphx_gpu PUBLIC -DMIGRAPHX_USE_ROCBLAS_FP8_API -DROCBLAS_BETA_FEATURES_API -DROCBLAS_NO_DEPRECATED_WARNINGS) - message(STATUS "MIGraphX is using Beta API of rocBLAS for FP8 computations") - else() - message(STATUS "rocBLAS does not have Fp8 Beta API") - endif() - - - target_link_libraries(migraphx_gpu PUBLIC roc::rocblas) -endif() - -if(MIGRAPHX_USE_HIPBLASLT) - target_link_libraries(migraphx_gpu PUBLIC roc::hipblaslt) -endif() - -if(WIN32) - # Temporary workaround on rocMLIR not exporting correctly libraries it depends on. - target_link_libraries(migraphx_gpu PRIVATE ntdll) -endif() - -target_link_libraries(migraphx_gpu PUBLIC migraphx) -if(NOT MIGRAPHX_USE_MIOPEN AND NOT MIGRAPHX_USE_ROCBLAS) - target_link_libraries(migraphx_gpu PUBLIC migraphx_device) -else() - target_link_libraries(migraphx_gpu PRIVATE migraphx_device) -endif() -target_link_libraries(migraphx_gpu PRIVATE migraphx_kernels) -if(MIGRAPHX_USE_COMPOSABLEKERNEL) - target_link_libraries(migraphx_gpu PRIVATE composable_kernel::jit_library) - target_compile_definitions(migraphx_gpu PRIVATE MIGRAPHX_USE_COMPOSABLEKERNEL=1) -endif() - -add_subdirectory(driver) -add_subdirectory(hiprtc) - -rocm_install_targets( - PRIVATE - TARGETS migraphx_gpu migraphx_device compile_for_gpu - INCLUDE - ${CMAKE_CURRENT_SOURCE_DIR}/include -) diff --git a/docker/rocm/migraphx/targets/gpu/abs.cpp b/docker/rocm/migraphx/targets/gpu/abs.cpp deleted file mode 100644 index 8cd0a1d8b..000000000 --- a/docker/rocm/migraphx/targets/gpu/abs.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_MIOPEN -shape miopen_abs::compute_shape(const std::vector& inputs) const -{ - check_shapes{inputs, *this}.has(2).packed(); - return inputs.at(0); -} - -argument miopen_abs::compute(context& ctx, - const shape& output_shape, - const std::vector& args) const -{ - float alpha = 1; - float beta = 0; - auto x_desc = make_tensor(args[0].get_shape()); - auto y_desc = make_tensor(output_shape); - miopenActivationForward(ctx.get_stream().get_miopen(), - ad.get(), - &alpha, - x_desc.get(), - args[0].implicit(), - &beta, - y_desc.get(), - args[1].implicit()); - - return args[1]; -} - -void miopen_abs::finalize(context&, const shape&, const std::vector&) { ad = make_abs(); } -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/allocation_model.cpp b/docker/rocm/migraphx/targets/gpu/allocation_model.cpp deleted file mode 100644 index e5fd2cc27..000000000 --- a/docker/rocm/migraphx/targets/gpu/allocation_model.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -std::string gpu_allocation_model::name() const { return "hip::allocate"; } -operation gpu_allocation_model::allocate(const shape& s) const -{ - return make_op(name(), {{"shape", to_value(s)}}); -} - -operation gpu_allocation_model::preallocate(const shape& s, const std::string& id) const -{ - return make_op("hip::hip_allocate_memory", {{"shape", to_value(s)}, {"id", id}}); -} - -std::string gpu_allocation_model::copy() const { return "hip::copy"; } - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/analyze_streams.cpp b/docker/rocm/migraphx/targets/gpu/analyze_streams.cpp deleted file mode 100644 index e08c89d82..000000000 --- a/docker/rocm/migraphx/targets/gpu/analyze_streams.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct hip_stream_model -{ - std::size_t max_stream = 0; - std::unordered_map ins2stream{}; - std::size_t get_nstream() const { return max_stream + 1; } - std::size_t get_stream(migraphx::instruction_ref ins) const { return ins2stream.at(ins); } - std::size_t get_event_id(migraphx::instruction_ref ins) const - { - auto v = ins->get_operator().to_value(); - return v["event"].to(); - } - bool has_stream(migraphx::instruction_ref ins) const { return ins2stream.count(ins) > 0; } - bool is_record(migraphx::instruction_ref ins) const - { - return ins->name() == "gpu::record_event"; - } - bool is_wait(migraphx::instruction_ref ins) const { return ins->name() == "gpu::wait_event"; } -}; - -stream_model make_stream_model(const module& m) -{ - hip_stream_model hsm; - std::size_t stream = 0; - for(auto ins : iterator_for(m)) - { - if(ins->name() == "gpu::set_stream") - { - auto v = ins->get_operator().to_value(); - stream = v["stream"].to(); - hsm.max_stream = std::max(stream, hsm.max_stream); - } - if(ins->get_operator().is_context_free()) - continue; - if(contains({"hip::hip_allocate_memory", "hip::hip_copy_literal", "@param"}, ins->name())) - continue; - hsm.ins2stream[ins] = stream; - } - return hsm; -} - -std::vector analyze_streams(const module& m) -{ - return migraphx::analyze_streams(m, make_stream_model(m)); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/argmax.cpp b/docker/rocm/migraphx/targets/gpu/argmax.cpp deleted file mode 100644 index b5f720295..000000000 --- a/docker/rocm/migraphx/targets/gpu/argmax.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_argmax::compute_shape(const std::vector& inputs) const -{ - check_shapes{inputs, *this}.has(2); - return op.normalize_compute_shape({inputs.at(0)}); -} - -argument hip_argmax::compute(context& ctx, const shape&, const std::vector& args) const -{ - auto n_dim = args.front().get_shape().lens().size(); - int64_t tuned_axis = tune_axis(n_dim, op.axis, op.name()); - device::argmax( - ctx.get_stream().get(), args.back(), args.front(), tuned_axis, op.select_last_index); - return args.back(); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/argmin.cpp b/docker/rocm/migraphx/targets/gpu/argmin.cpp deleted file mode 100644 index 02c44e29f..000000000 --- a/docker/rocm/migraphx/targets/gpu/argmin.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_argmin::compute_shape(const std::vector& inputs) const -{ - check_shapes{inputs, *this}.has(2); - return op.normalize_compute_shape({inputs.at(0)}); -} - -argument hip_argmin::compute(context& ctx, const shape&, const std::vector& args) const -{ - auto n_dim = args.front().get_shape().lens().size(); - int64_t tuned_axis = tune_axis(n_dim, op.axis, op.name()); - device::argmin( - ctx.get_stream().get(), args.back(), args.front(), tuned_axis, op.select_last_index); - return args.back(); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/code_object_op.cpp b/docker/rocm/migraphx/targets/gpu/code_object_op.cpp deleted file mode 100644 index 98a580dc4..000000000 --- a/docker/rocm/migraphx/targets/gpu/code_object_op.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_REGISTER_OP(code_object_op); - -shape code_object_op::compute_shape(std::vector inputs) const -{ - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [](const shape& s) { - return s.normalize_standard(); - }); - auto einputs = expected_inputs; - std::transform(einputs.begin(), einputs.end(), einputs.begin(), [](const shape& s) { - return s.normalize_standard(); - }); - if(not migraphx::equal(flatten(einputs), flatten(inputs), &shape::is_compatible)) - MIGRAPHX_THROW("Input shapes have changed: [" + to_string_range(einputs) + "] -> [" + - to_string_range(inputs) + "]"); - return output; -} -argument -code_object_op::compute(context& ctx, const shape&, const std::vector& args) const -{ - auto fargs = flatten(args); - std::vector kargs(fargs.size()); - std::transform( - fargs.begin(), fargs.end(), kargs.begin(), [](const argument& a) { return a.data(); }); - auto [start, stop] = ctx.get_perf_events(); - k.launch(ctx.get_stream().get(), global, local, std::move(kargs), start, stop); - return args[get_output_arg(args.size())]; -} -void code_object_op::finalize(context&, const shape&, const std::vector&) -{ - assert(not code_object.empty()); - k = kernel(code_object, symbol_name); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compile_gen.cpp b/docker/rocm/migraphx/targets/gpu/compile_gen.cpp deleted file mode 100644 index 82ff3c4a2..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_gen.cpp +++ /dev/null @@ -1,576 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace gen { - -static std::vector vector_sizes(const std::vector& inputs) -{ - // If all inputs are half then only use half2 - if(std::all_of(inputs.begin(), inputs.end(), [](const auto& s) { - return s.type() == shape::half_type; - })) - return {2}; - return {4, 2}; -} - -vectorize vectorize::elements(std::size_t axis, - const std::vector& inputs, - const std::vector& sizes) -{ - // disable vectorization for fp8 types - if(std::any_of(inputs.begin(), inputs.end(), [&](auto ishape) { - return contains(fp8_types{}.get(), ishape.type()); - })) - return {1, axis}; - if(std::all_of( - inputs.begin(), inputs.end(), [&](const auto& s) { return s.lens()[axis] == 1; })) - return {1, axis}; - std::vector max_vec_size; - std::transform(inputs.begin(), - inputs.end(), - std::back_inserter(max_vec_size), - [&](const auto& input) -> std::size_t { - auto stride = input.strides()[axis]; - auto len = input.lens()[axis]; - if(not contains({0, 1}, stride)) - return 1; - if(len == 1 and input.elements() > sizes.front()) - return sizes.front(); - auto it = std::find_if(sizes.begin(), sizes.end(), [&](auto vsize) { - // The len is divisible by the size and all the strides are divisible by - // the size - return (len % vsize) == 0 and - std::all_of( - input.strides().begin(), input.strides().end(), [&](auto i) { - return contains({0, 1}, i) or i % vsize == 0; - }); - }); - if(it != sizes.end()) - return *it; - return 1; - }); - return {*std::min_element(max_vec_size.begin(), max_vec_size.end()), axis}; -} - -vectorize vectorize::elements(context& ctx, std::size_t axis, const std::vector& inputs) -{ - // disable vectorization for fp8 types - if(std::any_of(inputs.begin(), inputs.end(), [&](auto ishape) { - return contains(fp8_types{}.get(), ishape.type()); - })) - return {1, axis}; - if(inputs.empty()) - return {1, axis}; - std::size_t n = std::max_element(inputs.begin(), - inputs.end(), - by(std::less<>{}, [](const auto& s) { return s.elements(); })) - ->elements(); - std::size_t max_global = ctx.get_current_device().get_cu_count() * - ctx.get_current_device().get_max_workitems_per_cu(); - std::size_t over = n / max_global; - bool broadcasted = - std::any_of(inputs.begin(), inputs.end(), [](const auto& s) { return s.broadcasted(); }); - std::vector sizes; - if(broadcasted and over > 8) - sizes.push_back(8); - if(over > 4) - sizes.push_back(4); - sizes.push_back(2); - return elements(axis, inputs, sizes); -} - -vectorize vectorize::elements(std::size_t axis, const std::vector& inputs) -{ - return elements(axis, inputs, vector_sizes(inputs)); -} - -std::string vectorize::str() const -{ - return "vectorize<" + to_string(size) + ", " + to_string(axis) + ">()"; -} - -preload preload::broadcasts(std::size_t axis, const std::vector& inputs) -{ - const std::size_t max_lds_bytes = 4096; - std::vector result(inputs.size()); - std::vector preloaded; - auto idxs = range(inputs.size()); - std::copy_if(idxs.begin(), idxs.end(), std::back_inserter(preloaded), [&](auto i) { - return inputs[i].strides()[axis] == 0; - }); - std::sort(preloaded.begin(), preloaded.end(), by(std::less<>{}, [&](auto i) { - return inputs[i].bytes(); - })); - - std::size_t bytes = 0; - for(auto i : preloaded) - { - const auto& input = inputs[i]; - bytes += input.bytes(); - if(bytes > max_lds_bytes) - break; - result[i] = true; - } - return {result}; -} - -std::string preload::str() const -{ - std::vector bool_strs; - std::transform(args.begin(), std::prev(args.end()), std::back_inserter(bool_strs), [](bool b) { - if(b) - return "true"; - return "false"; - }); - return "auto_preload(idx)"; -} - -bool preload::is_preloading() const -{ - return std::accumulate(args.begin(), args.end(), false, std::logical_or<>{}); -} - -static std::size_t integer_divide_ceil(std::size_t x, std::size_t y) -{ - return (x + y - std::size_t{1}) / y; -} - -static std::size_t compute_tile_factor(std::size_t r, std::size_t max_size = 64) -{ - std::size_t n = 1; - auto factors = make_array(2, 3, 5, 7, 11); - while(n < max_size) - { - // NOLINTNEXTLINE(readability-qualified-auto) - auto it = std::find_if(factors.begin(), factors.end(), [&](auto d) { return r % d == 0; }); - if(it == factors.end()) - break; - r /= *it; - n *= *it; - } - return n; -} - -tile tile::elements(const std::vector& inputs, std::size_t noutputs) -{ - tile result; - auto ndim = inputs.front().ndim(); - std::vector faxes; - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(faxes), MIGRAPHX_LIFT(find_fast_axis)); - result.axis = std::accumulate(faxes.begin(), faxes.end(), ndim, MIGRAPHX_LIFT(std::min)); - if(result.axis >= (ndim - 1)) - return {}; - auto select = [&](auto m) { - return [&, m](std::size_t faxis, shape input) { - if(input.broadcasted()) - return none; - if(faxis < (ndim - 1)) - return m; - return none; - }; - }; - std::transform(faxes.begin(), - faxes.end() - noutputs, - inputs.begin(), - std::back_inserter(result.args), - select(load)); - std::transform(faxes.end() - noutputs, - faxes.end(), - inputs.end() - noutputs, - std::back_inserter(result.args), - select(store)); - - auto nargs = std::count_if( - result.args.begin(), result.args.end(), [](auto m) { return m != mode::none; }); - // TODO: Handle tiling more than one arguments - if(nargs != 1) - return {}; - - const auto& s = inputs.front(); - auto dim1 = compute_tile_factor(s.lens()[result.axis]); - auto dim2 = compute_tile_factor(s.lens().back(), 4096 / dim1); - if(dim1 == 1 or dim2 == 1) - return {}; - - result.inner = s.lens(); - std::fill(result.inner.begin(), result.inner.end(), 1); - result.inner[result.axis] = dim1; - result.inner.back() = dim2; - - result.outer = s.lens(); - result.outer[result.axis] /= dim1; - result.outer.back() /= dim2; - - auto tile_size = dim1 * dim2; - result.ntiles = s.elements() / tile_size; - // equivalent to dim1 * (dim2 + 1) to avoid bank conflicts - auto tile_bytes = (tile_size + dim1) * s.type_size(); - if(tile_bytes > 65536) - return {}; - - result.block_size = std::min(256, integer_divide_ceil(tile_size / 4, 64) * 64); - return result; -} - -std::string tile::str() const -{ - if(args.empty()) - return "transform_args()"; - std::vector strs; - std::transform(args.begin(), args.end(), std::back_inserter(strs), [](mode m) { - switch(m) - { - case load: return "tile::load"; - case store: return "tile::store"; - case none: return "tile::none"; - } - MIGRAPHX_THROW("Invalid mode"); - }); - const std::string auto_tile = "auto_tile<${modes}>(${inner}, ${outer})"; - return interpolate_string(auto_tile, - {{"modes", join_strings(strs, ", ")}, - {"inner", generate_index_ints(inner)}, - {"outer", generate_index_ints(outer)}}); -} - -std::size_t find_fast_axis(const shape& input) -{ - if(input.scalar()) - return input.ndim() - 1; - if(input.broadcasted()) - { - auto stride_it = std::min_element( - input.strides().begin(), input.strides().end(), by(std::less<>{}, [](std::size_t i) { - if(i == 0) - return std::numeric_limits::max(); - return i; - })); - return stride_it - input.strides().begin(); - } - auto permutation = invert_permutation(find_permutation(input)); - auto it = std::max_element(permutation.begin(), permutation.end()); - return it - permutation.begin(); -} - -std::size_t find_fast_axis(const std::vector& inputs) -{ - auto permutation = invert_permutation(find_permutation(inputs)); - auto it = std::max_element(permutation.begin(), permutation.end()); - return it - permutation.begin(); -} - -std::string make_transformer_args(std::vector transformers) -{ - return join_strings(std::move(transformers), ", "); -} - -static void generate_pointwise(cpp_generator& gg, - const module& pm, - const std::string& name, - bool always_return_tuple = false) -{ - module m = pm; - run_passes(m, {rewrite_quantization{}, optimize_module{}}); - m.sort(); - cpp_generator g; - g.always_return_tuple(always_return_tuple); - g.fmap([](const std::string& fname) { return "migraphx::" + fname; }); - g.add_point_op("where", "${function:where}(${0}, ${1}, ${2})"); - g.add_point_op("prelu", "${function:where}(${0} < 0, ${0} * ${1}, ${0})"); - g.add_point_op("sign", "${function:where}(${0} > 0, 1, ${function:where}(${0} < 0, -1, 0))"); - g.add_point_op("equal", "migraphx::abs(${0} == ${1})"); - g.add_point_op("less", "migraphx::abs(${0} < ${1})"); - g.add_point_op("greater", "migraphx::abs(${0} > ${1})"); - g.add_point_op("not", "migraphx::abs(not ${0})"); - // Add explict conversions - g.fresult( - [](const shape& s) { return "migraphx::convert<" + shape::cpp_type(s.type()) + ">"; }); - gg.create_function(g.generate_module(m) - .set_attributes({"__device__", "__attribute__((const))"}) - .set_generic_types(m) - .set_name(name)); -} -std::string generate_pointwise(const module& pm, const std::string& name, bool always_return_tuple) -{ - cpp_generator g; - generate_pointwise(g, pm, name, always_return_tuple); - return g.str(); -} - -std::string reduce_op::str() const -{ - return write + "(r.reduce(" + reduction + ", " + init + ", " + read + ")(" + - join_strings(inputs, ", ") + "))"; -} -void reduce_op::set(const std::string& name, const shape& input, const shape& output) -{ - assert(input.type() != shape::tuple_type); - assert(output.type() != shape::tuple_type); - if(name == "reduce_sum") - { - reduction = "op::sum{}"; - } - else if(name == "reduce_mean") - { - auto reduce_elements = input.elements() / output.elements(); - auto reduce_type = input.type(); - reduction = "op::sum{}"; - std::string mean = "op::mean<" + std::to_string(reduce_elements) + ">{}"; - // Use float accumulator when reduction size is too large for half - if(reduce_type == shape::half_type and reduce_elements > 16384) - read = "compose(" + mean + ", op::convert_to{})"; - else if(contains({shape::float_type, shape::half_type, shape::double_type}, reduce_type)) - read = mean; - else - write = mean; - } - else if(name == "reduce_max") - { - reduction = "op::max{}"; - init = "lowest{}"; - } - else if(name == "reduce_min") - { - reduction = "op::min{}"; - init = "highest{}"; - } - else if(name == "reduce_prod") - { - reduction = "op::product{}"; - init = "1"; - } - else if(name == "reduce_any") - { - reduction = "op::logical_or{}"; - init = "bool{false}"; - } - else if(name == "reduce_all") - { - reduction = "op::logical_and{}"; - init = "bool{true}"; - } - else - { - MIGRAPHX_THROW("Unsupported reduce"); - } -} - -void reduce_op::set(instruction_ref ins, const operation& op) -{ - if(op.name() == "gpu::parallel_reduce") - { - auto rop = from_value(op.to_value().at("op")); - auto input = ins->inputs().front()->get_shape(); - auto output = ins->get_shape().sub_shapes().front(); - set(rop.name(), input, output); - read = "compose(array_apply(" + read + "), MIGRAPHX_LIFT(make_array))"; - } - else - { - set(op.name(), ins->inputs().front()->get_shape(), ins->get_shape()); - } -} -std::string reduce_op::generate(instruction_ref ins, const std::vector& x) -{ - reduce_op r{x}; - r.set(ins, ins->get_operator()); - return r.str(); -} - -static bool use_lazy_inner(instruction_ref ins) -{ - if(ins->outputs().size() != 1) - return false; - // When the inputs are broadcasted, it means the lambda will capture SGPRs - // when doing block/wave reduction. This can cause register spilling in - // the compiler when the lambda is evaluated at a later time although it - // shouldn't. Instead, use `inner` to workaround this issue in the - // compiler. - if(std::any_of(ins->inputs().begin(), ins->inputs().end(), [](instruction_ref input) { - return input->get_shape().broadcasted(); - })) - return false; - auto output = ins->outputs().front(); - return contains(output->name(), "reduce") or output->name() == "@return"; -} - -void preload_params(module& m) -{ - for(auto ins : iterator_for(m)) - { - if(ins->name() != "@param") - continue; - if(ins->outputs().size() <= 1) - continue; - auto id = m.insert_instruction(std::next(ins), make_op("identity"), ins); - m.replace_instruction(ins, id); - } -} - -std::string generate_reduce(module m, const std::string& name) -{ - preload_params(m); - run_passes(m, {optimize_module{}, prepare_reduce{}, optimize_module{}}); - m.sort(); - cpp_generator g; - g.always_return_tuple(); - auto param_shapes = m.get_parameter_shapes(); - auto max_shape = - std::max_element(param_shapes.begin(), - param_shapes.end(), - by(std::less<>{}, [](const auto& p) { return p.second.elements(); })); - auto ilens = max_shape->second.lens(); - std::size_t i = 0; - auto f = g.generate_module(m, [&](instruction_ref ins, const auto& names) { - if(contains(ins->name(), "reduce")) - { - return reduce_op::generate(ins, cpp_generator::to_args(ins->inputs(), names)); - } - if(ins->name() == "pointwise") - { - auto pointwise_name = "pointwise" + std::to_string(i); - i++; - generate_pointwise(g, *ins->module_inputs().front(), pointwise_name); - std::vector tensors; - std::copy_if(ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(tensors), - [&](auto input) { - return input->get_shape().lens() == ilens and - not input->get_shape().broadcasted(); - }); - auto inner_names = names; - for(auto input : ins->inputs()) - { - if(input->name() != "@param") - continue; - if(contains(tensors, input)) - continue; - inner_names[input] += "[out_idx]"; - } - for(auto input : tensors) - inner_names[input] += "_lambda_param"; - auto call_function = - pointwise_name + "(" + - join_strings(cpp_generator::to_args(ins->inputs(), inner_names), ", ") + ")"; - if(tensors.empty()) - return call_function; - const std::string inner_template = - "r.${inner}([=](${params}) { return ${call}; })(${args})"; - std::string inner_name = use_lazy_inner(ins) ? "lazy_inner" : "inner"; - auto args = cpp_generator::to_args(tensors, names); - auto params = cpp_generator::to_args(tensors, inner_names); - std::transform( - params.begin(), params.end(), params.begin(), [](auto s) { return "auto " + s; }); - return interpolate_string(inner_template, - {{"inner", inner_name}, - {"params", join_strings(params, ", ")}, - {"args", join_strings(args, ", ")}, - {"call", call_function}}); - } - if(ins->name() == "multibroadcast") - { - return names.at(ins->inputs().front()); - } - if(ins->name() == "get_tuple_elem") - { - const auto& x = names.at(ins->inputs().front()); - auto index = ins->get_operator().to_value()["index"].to(); - return interpolate_string("${x}[${index}]", - {{"x", x}, {"index", std::to_string(index)}}); - } - if(ins->name() == "identity") - { - const auto& x = names.at(ins->inputs().front()); - return "r.inner(op::id{})(" + x + ")"; - } - MIGRAPHX_THROW("Unknown operator: " + ins->name()); - }); - f.set_attributes({"__device__", "__attribute__((const))"}).set_generic_types(m).set_name(name); - f.add_generic_param("r"); - f.add_generic_param("out_idx"); - f.unused_param("out_idx"); - g.create_function(f); - return g.str(); -} - -static std::vector get_op_names(const module& m) -{ - std::vector result; - for(auto& ins : m) - { - if(starts_with(ins.name(), "@")) - continue; - if(contains({"multibroadcast", "contiguous", "identity"}, ins.name())) - continue; - if(ins.name() == "pointwise") - { - auto names = get_op_names(*ins.module_inputs().front()); - result.insert(result.end(), names.begin(), names.end()); - } - else - { - result.push_back(ins.name()); - } - } - return result; -} - -std::string generate_name_from_ops(const module& m, const std::string& postname) -{ - auto op_names = get_op_names(m); - if(not postname.empty()) - op_names.push_back(postname); - if(op_names.empty()) - return "noop"; - return join_strings(op_names, "_"); -} - -} // namespace gen -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compile_hip.cpp b/docker/rocm/migraphx/targets/gpu/compile_hip.cpp deleted file mode 100644 index 58b518725..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_hip.cpp +++ /dev/null @@ -1,406 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef MIGRAPHX_USE_HIPRTC -#include -#include -#include -#include -#include -#include -#include -#include -#include -#else -#include -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DEBUG); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DEBUG_SYM); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_OPTIMIZE); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DUMP_ASM); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_DUMP_SRC); - -#ifdef MIGRAPHX_USE_HIPRTC - -std::string hiprtc_error(hiprtcResult err, const std::string& msg) -{ - return "hiprtc: " + (hiprtcGetErrorString(err) + (": " + msg)); -} - -void hiprtc_check_error(hiprtcResult err, const std::string& msg, const std::string& ctx) -{ - if(err != HIPRTC_SUCCESS) - throw make_exception(ctx, hiprtc_error(err, msg)); -} - -// NOLINTNEXTLINE -#define MIGRAPHX_HIPRTC(...) \ - hiprtc_check_error(__VA_ARGS__, #__VA_ARGS__, MIGRAPHX_MAKE_SOURCE_CTX()) - -#define MIGRAPHX_HIPRTC_THROW(error, msg) MIGRAPHX_THROW(hiprtc_error(error, msg)) - -// Workaround hiprtc's broken API -void hiprtc_program_destroy(hiprtcProgram prog) { hiprtcDestroyProgram(&prog); } -using hiprtc_program_ptr = MIGRAPHX_MANAGE_PTR(hiprtcProgram, hiprtc_program_destroy); - -template -hiprtc_program_ptr hiprtc_program_create(Ts... xs) -{ - hiprtcProgram prog = nullptr; - auto result = hiprtcCreateProgram(&prog, xs...); - hiprtc_program_ptr p{prog}; - if(result != HIPRTC_SUCCESS) - MIGRAPHX_HIPRTC_THROW(result, "Create program failed."); - return p; -} - -struct hiprtc_program -{ - struct string_array - { - std::deque strings{}; - std::vector c_strs{}; - - string_array() {} - string_array(const string_array&) = delete; - - std::size_t size() const { return strings.size(); } - - const char** data() { return c_strs.data(); } - - void push_back(std::string s) - { - strings.push_back(std::move(s)); - c_strs.push_back(strings.back().c_str()); - } - }; - - hiprtc_program_ptr prog = nullptr; - string_array headers{}; - string_array include_names{}; - std::string cpp_src = ""; - std::string cpp_name = ""; - - hiprtc_program(const std::string& src, const std::string& name = "main.cpp") - : cpp_src(src), cpp_name(name) - { - create_program(); - } - - hiprtc_program(std::vector srcs) - { - for(auto&& src : srcs) - { - if(ends_with(src.path, ".cpp")) - { - cpp_src = std::move(src.content); - cpp_name = std::move(src.path); - } - else - { - headers.push_back(std::move(src.content)); - include_names.push_back(std::move(src.path)); - } - } - create_program(); - } - - void create_program() - { - assert(not cpp_src.empty()); - assert(not cpp_name.empty()); - assert(headers.size() == include_names.size()); - prog = hiprtc_program_create(cpp_src.c_str(), - cpp_name.c_str(), - headers.size(), - headers.data(), - include_names.data()); - } - - void compile(const std::vector& options, bool quiet = false) const - { - if(enabled(MIGRAPHX_TRACE_HIPRTC{})) - std::cout << "hiprtc " << join_strings(options, " ") << " " << cpp_name << std::endl; - std::vector c_options; - std::transform(options.begin(), - options.end(), - std::back_inserter(c_options), - [](const std::string& s) { return s.c_str(); }); - auto result = hiprtcCompileProgram(prog.get(), c_options.size(), c_options.data()); - auto prog_log = log(); - if(not prog_log.empty() and not quiet) - { - std::cerr << prog_log << std::endl; - } - if(result != HIPRTC_SUCCESS) - MIGRAPHX_HIPRTC_THROW(result, "Compilation failed."); - } - - std::string log() const - { - std::size_t n = 0; - MIGRAPHX_HIPRTC(hiprtcGetProgramLogSize(prog.get(), &n)); - if(n == 0) - return {}; - std::string buffer(n, '\0'); - MIGRAPHX_HIPRTC(hiprtcGetProgramLog(prog.get(), buffer.data())); - assert(buffer.back() != 0); - return buffer; - } - - std::vector get_code_obj() const - { - std::size_t n = 0; - MIGRAPHX_HIPRTC(hiprtcGetCodeSize(prog.get(), &n)); - std::vector buffer(n); - MIGRAPHX_HIPRTC(hiprtcGetCode(prog.get(), buffer.data())); - return buffer; - } -}; - -std::vector> compile_hip_src_with_hiprtc(std::vector srcs, - const std::vector& params, - const std::string& arch) -{ - hiprtc_program prog(std::move(srcs)); - auto options = params; - options.push_back("-DMIGRAPHX_USE_HIPRTC=1"); - if(enabled(MIGRAPHX_GPU_DEBUG{})) - options.push_back("-DMIGRAPHX_DEBUG"); - if(std::none_of(options.begin(), options.end(), [](const std::string& s) { - return starts_with(s, "--std=") or starts_with(s, "-std="); - })) - options.push_back("-std=c++17"); - options.push_back("-fno-gpu-rdc"); - options.push_back("-O" + string_value_of(MIGRAPHX_GPU_OPTIMIZE{}, "3")); - options.push_back("-Wno-cuda-compat"); - options.push_back("--offload-arch=" + arch); - prog.compile(options); - return {prog.get_code_obj()}; -} - -bool hip_has_flags(const std::vector& flags) -{ - hiprtc_program prog{" "}; - - std::string src = " "; - src_file input{"main.cpp", src}; - std::vector srcs = {input}; - - try - { - std::string arch = "gfx900"; - compile_hip_src(srcs, flags, arch); - return true; - } - catch(...) - { - return false; - } -} - -std::vector> compile_hip_src(const std::vector& srcs, - const std::vector& params, - const std::string& arch) -{ - std::vector hsrcs{srcs.begin(), srcs.end()}; - if(enabled(MIGRAPHX_GPU_DUMP_SRC{})) - { - for(const auto& src : srcs) - { - if(src.path.extension() != ".cpp") - continue; - std::cout << std::string(src.content) << std::endl; - } - } - - auto fname = make_executable_filename("migraphx-hiprtc-driver"); - auto p = dynamic_loader::path(&compile_hip_src_with_hiprtc); - auto driver = p.parent_path() / fname; - - bool found = fs::exists(driver); - if(not found) - { - driver = p.parent_path().parent_path() / "bin" / fname; - found = fs::exists(driver); - } - - if(found) - { - value v; - v["srcs"] = to_value(hsrcs); - v["params"] = to_value(params); - v["arch"] = to_value(arch); - - tmp_dir td{}; - auto out = td.path / "output"; - - process(driver, {quote_string(out.string())}).write([&](auto writer) { - to_msgpack(v, writer); - }); - if(fs::exists(out)) - return {read_buffer(out)}; - } - return compile_hip_src_with_hiprtc(std::move(hsrcs), params, arch); -} - -#else // MIGRAPHX_USE_HIPRTC - -std::vector> -compile_hip_src_with_hiprtc(std::vector, // NOLINT - const std::vector&, // NOLINT - const std::string&) -{ - MIGRAPHX_THROW("Not using hiprtc"); -} - -bool is_hip_clang_compiler() -{ - static const auto result = fs::path{MIGRAPHX_HIP_COMPILER}.stem() == "clang++"; - return result; -} - -#ifdef MIGRAPHX_HIP_COMPILER_LAUNCHER - -bool has_compiler_launcher() -{ - static const auto result = fs::exists(MIGRAPHX_HIP_COMPILER_LAUNCHER); - return result; -} - -#endif - -src_compiler assemble(src_compiler compiler) -{ - compiler.out_ext = ".S"; - std::replace(compiler.flags.begin(), compiler.flags.end(), "-c", "-S"); - return compiler; -} - -std::vector> compile_hip_src(const std::vector& srcs, - const std::vector& params, - const std::string& arch) -{ - assert(not srcs.empty()); - - if(not is_hip_clang_compiler()) - MIGRAPHX_THROW("Unknown hip compiler: " MIGRAPHX_HIP_COMPILER); - - src_compiler compiler; - compiler.flags = params; - compiler.compiler = MIGRAPHX_HIP_COMPILER; -#ifdef MIGRAPHX_HIP_COMPILER_LAUNCHER - if(has_compiler_launcher()) - compiler.launcher = MIGRAPHX_HIP_COMPILER_LAUNCHER; -#endif - - if(std::none_of(params.begin(), params.end(), [](const std::string& s) { - return starts_with(s, "--std=") or starts_with(s, "-std="); - })) - compiler.flags.emplace_back("--std=c++17"); - compiler.flags.emplace_back(" -fno-gpu-rdc"); - if(enabled(MIGRAPHX_GPU_DEBUG_SYM{})) - compiler.flags.emplace_back("-g"); - compiler.flags.emplace_back("-c"); - compiler.flags.emplace_back("--offload-arch=" + arch); - compiler.flags.emplace_back("--cuda-device-only"); - compiler.flags.emplace_back("-O" + string_value_of(MIGRAPHX_GPU_OPTIMIZE{}, "3") + " "); - - if(enabled(MIGRAPHX_GPU_DEBUG{})) - compiler.flags.emplace_back("-DMIGRAPHX_DEBUG"); - - compiler.flags.emplace_back("-Wno-unused-command-line-argument"); - compiler.flags.emplace_back("-Wno-cuda-compat"); - compiler.flags.emplace_back(MIGRAPHX_HIP_COMPILER_FLAGS); - - if(enabled(MIGRAPHX_GPU_DUMP_SRC{})) - { - for(const auto& src : srcs) - { - if(src.path.extension() != ".cpp") - continue; - std::cout << std::string(src.content) << std::endl; - } - } - - if(enabled(MIGRAPHX_GPU_DUMP_ASM{})) - { - - std::cout << assemble(compiler).compile(srcs).data() << std::endl; - } - - return {compiler.compile(srcs)}; -} - -bool hip_has_flags(const std::vector& flags) -{ - src_compiler compiler; - compiler.compiler = MIGRAPHX_HIP_COMPILER; - compiler.flags = flags; - compiler.flags.emplace_back("-x hip"); - compiler.flags.emplace_back("-c"); - compiler.flags.emplace_back("--offload-arch=gfx900"); - compiler.flags.emplace_back("--cuda-device-only"); - - std::string src; - src_file input{"main.cpp", src}; - - try - { - compiler.compile({input}); - return true; - } - catch(...) - { - return false; - } -} - -#endif // MIGRAPHX_USE_HIPRTC - -std::string enum_params(std::size_t count, std::string param) -{ - std::vector items(count); - transform(range(count), items.begin(), [&](auto i) { return param + std::to_string(i); }); - return join_strings(items, ","); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compile_hip_code_object.cpp b/docker/rocm/migraphx/targets/gpu/compile_hip_code_object.cpp deleted file mode 100644 index dfd18ad7d..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_hip_code_object.cpp +++ /dev/null @@ -1,215 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -std::string generate_make_shape(const shape& s) -{ - return "make_shape(" + generate_index_ints(s.lens()) + ", " + generate_index_ints(s.strides()) + - ")"; -} - -static const char* const make_tensor_template = R"__migraphx__( -template<> -struct make_tensor<${n}> -{ - static __device__ auto apply(void* __restrict__ p) - { - return make_tensor_view(reinterpret_cast<${type}* __restrict__>(p), make_shape(${lens}, ${strides})); - } -}; -)__migraphx__"; - -std::string generate_make_tensor(std::size_t n, const shape& s) -{ - return interpolate_string(make_tensor_template, - {{"n", std::to_string(n)}, - {"type", shape::cpp_type(s.type())}, - {"lens", generate_index_ints(s.lens())}, - {"strides", generate_index_ints(s.strides())}}); -} - -std::string generate_args_hpp(const std::vector& inputs) -{ - std::string inner; - for(std::size_t i = 0; i < inputs.size(); i++) - { - inner += generate_make_tensor(i, inputs[i]); - } - const std::string args_hpp = R"__migraphx__( -#ifndef MIGRAPHX_GUARD_AUTO_ARGS_HPP -#define MIGRAPHX_GUARD_AUTO_ARGS_HPP - -#include -#include -#include - -namespace migraphx { - -__content__ - -} // namespace migraphx -#endif -)__migraphx__"; - return replace_string(args_hpp, "__content__", inner); -} - -static std::vector get_compiler_warnings() -{ - std::vector warnings = { - "-Weverything", - "-Wno-c++98-compat", - "-Wno-c++98-compat-pedantic", - "-Wno-conversion", - "-Wno-double-promotion", - "-Wno-exit-time-destructors", - "-Wno-extra-semi", - "-Wno-extra-semi-stmt", - "-Wno-float-conversion", - "-Wno-gnu-anonymous-struct", - "-Wno-gnu-zero-variadic-macro-arguments", - "-Wno-missing-prototypes", - "-Wno-nested-anon-types", - "-Wno-padded", - "-Wno-shorten-64-to-32", - "-Wno-sign-conversion", - "-Wno-sign-compare", - "-Wno-unused-command-line-argument", - "-Wno-weak-vtables", - "-Wno-c99-extensions", - }; - - if(hip_has_flags({"-Werror", "-Wunsafe-buffer-usage"})) - warnings.push_back("-Wno-unsafe-buffer-usage"); - return warnings; -} - -const std::vector& compiler_warnings() -{ - static std::vector warnings = get_compiler_warnings(); - return warnings; -} - -void hip_compile_options::set_launch_params( - const value& v, - const std::function& compute_global, - std::size_t default_local) -{ - local = v.get("local", default_local); - if(v.contains("global")) - global = v.at("global").to(); - else - global = compute_global(local); -} - -static bool hip_accept_non_uniform_wg() -{ - static bool non_uniform_wg = hip_has_flags({"-fno-offload-uniform-block"}); - return non_uniform_wg; -} - -std::function -compute_global_for(context& ctx, std::size_t n, std::size_t over) -{ - assert(over > 0); - std::size_t max_global = ctx.get_current_device().get_cu_count() * - ctx.get_current_device().get_max_workitems_per_cu(); - return [n, over, max_global](std::size_t local) { - std::size_t num_elements = n; - if(not hip_accept_non_uniform_wg()) - { - num_elements = (1 + (n - 1) / local) * local; - } - std::size_t groups = 1 + (num_elements - 1) / local; - std::size_t max_blocks = max_global / local; - std::size_t nglobal = std::min(max_blocks * over, groups) * local; - return std::min(nglobal, num_elements); - }; -} - -std::size_t compute_block_size(context& ctx, std::size_t n, std::size_t max_block_size) -{ - const std::size_t min_block_size = ctx.get_current_device().get_wavefront_size(); - auto block_size = (((n - 1) / min_block_size + 1)) * min_block_size; - return std::min(std::max(min_block_size, block_size), max_block_size); -} - -operation -compile_hip_code_object(context& ctx, const std::string& content, hip_compile_options options) -{ - assert(options.global > 0); - assert(options.local > 0); - assert(not options.inputs.empty()); - assert(options.inputs.size() == options.virtual_inputs.size() or - options.virtual_inputs.empty()); - std::vector srcs = options.additional_src_files; - static auto kernels{::migraphx_kernels()}; - std::transform( - kernels.begin(), - kernels.end(), - std::back_inserter(srcs), - [](const std::pair& elem) { return src_file{elem}; }); - srcs.emplace_back("main.cpp", content); - auto args_hpp = - generate_args_hpp(options.virtual_inputs.empty() ? options.inputs : options.virtual_inputs); - srcs.emplace_back("args.hpp", args_hpp); - - if(options.global % options.local != 0 and hip_accept_non_uniform_wg()) - options.emplace_param("-fno-offload-uniform-block"); - else - assert(options.global % options.local == 0); - - options.emplace_param("-DMIGRAPHX_NGLOBAL=" + std::to_string(options.global)); - options.emplace_param("-DMIGRAPHX_NLOCAL=" + std::to_string(options.local)); - options.emplace_param("-DMIGRAPHX_WAVEFRONTSIZE=" + - std::to_string(ctx.get_current_device().get_wavefront_size())); - const auto& warnings = compiler_warnings(); - options.params.insert(options.params.end(), warnings.begin(), warnings.end()); - options.emplace_param("-ftemplate-backtrace-limit=0"); - options.emplace_param("-Werror"); - auto cos = compile_hip_src(srcs, options.params, get_device_name()); - if(cos.size() != 1) - MIGRAPHX_THROW("No code object"); - return code_object_op{value::binary{cos.front()}, - options.kernel_name, - options.global, - options.local, - options.inputs, - options.output, - options.output_arg}; -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compile_hipblaslt.cpp b/docker/rocm/migraphx/targets/gpu/compile_hipblaslt.cpp deleted file mode 100644 index c320e6b7d..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_hipblaslt.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#if MIGRAPHX_USE_HIPBLASLT -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -static size_t compile(migraphx::context& ctx, operation& op, instruction_ref ins) -{ - auto v = op.compile(ctx, ins->get_shape(), to_shapes(ins->inputs())); - return v.get("workspace", 0); -} - -void compile_hipblaslt::apply(module& m) const -{ - assert(ctx); - for(auto ins : iterator_for(m)) - { - if(ins->name() != "gpu::hipblaslt_op") - continue; - auto op = any_cast(ins->get_operator()).op; - auto inputs = ins->inputs(); - - std::size_t ws = hipblaslt_workspace_size; - - auto alloc = m.insert_instruction( - ins, make_op("allocate", {{"shape", to_value(shape{shape::uint8_type, {ws}})}})); - inputs.insert(std::prev(inputs.end()), alloc); - m.replace_instruction(ins, op, inputs); - - // Calculate workspace size - ws = compile(*ctx, op, ins); - auto alloc_after = m.insert_instruction( - ins, make_op("allocate", {{"shape", to_value(shape{shape::uint8_type, {ws}})}})); - - // Replace the workspace size with actual worksapce size needed. - auto it = std::find(inputs.begin(), inputs.end(), alloc); - if(it != inputs.end()) - { - *it = alloc_after; // Replace `alloc` with `alloc_after` - } - m.replace_instruction(ins, op, inputs); - } -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_USE_HIPBLASLT diff --git a/docker/rocm/migraphx/targets/gpu/compile_miopen.cpp b/docker/rocm/migraphx/targets/gpu/compile_miopen.cpp deleted file mode 100644 index 583601bdd..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_miopen.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct miopen_op -{ - operation op = op::identity{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op")); - } - - std::string name() const { return "gpu::miopen_op"; } - - shape compute_shape(std::vector inputs) const - { - inputs.push_back(inputs.back()); - return op.compute_shape(inputs); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -MIGRAPHX_REGISTER_OP(miopen_op); - -std::size_t compile_miopen::compile(operation& op, instruction_ref ins) const -{ - auto v = op.compile(*ctx, ins->get_shape(), to_shapes(ins->inputs())); - return v.get("workspace", 0); -} - -void compile_miopen::apply(module& m) const -{ - assert(ctx); - for(auto ins : iterator_for(m)) - { - if(ins->name() != "gpu::miopen_op") - continue; - auto op = any_cast(ins->get_operator()).op; - std::size_t ws = 0; - ws = compile(op, ins); - auto inputs = ins->inputs(); - auto alloc = m.insert_instruction( - ins, make_op("allocate", {{"shape", to_value(shape{shape::int8_type, {ws}})}})); - inputs.insert(std::prev(inputs.end()), alloc); - - m.replace_instruction(ins, op, inputs); - } -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compile_ops.cpp b/docker/rocm/migraphx/targets/gpu/compile_ops.cpp deleted file mode 100644 index cc5a7fc24..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_ops.cpp +++ /dev/null @@ -1,332 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_GPU_COMPILE_PARALLEL); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_BENCHMARKING); - -struct precompile_op -{ - operation op = op::identity{}; - std::size_t additional_args = 1; - bool ignore_modules = false; - std::optional output_shape = nullopt; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op"), - f(self.additional_args, "additional_args"), - f(self.ignore_modules, "ignore_modules"), - f(self.output_shape, "output_shape")); - } - - std::string name() const { return "gpu::precompile_op"; } - - shape compute_shape(std::vector inputs, const std::vector& mods) const - { - // Pop off additional args - inputs.resize(inputs.size() - additional_args); - if(output_shape.has_value()) - return output_shape.value(); - if(ignore_modules) - return op.compute_shape(inputs); - return op.compute_shape(inputs, mods); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -MIGRAPHX_REGISTER_OP(precompile_op); - -struct compiled_result -{ - compiler_replace replace; - instruction_ref ins; - - friend std::ostream& operator<<(std::ostream& os, const compiled_result& cr) - { - cr.replace.trace(os, cr.ins); - return os; - } -}; - -struct compile_plan -{ - context* ctx; - operation preop; - instruction_ref ins; - optional config = nullopt; - std::vector> results = {}; - void update_config(bool exhaustive) - { - config = get_tuning_config(*ctx, ins, preop, exhaustive); - } - template - void insert_compiles(Vector& compiles, const value& solution, std::size_t i) - { - compiles.emplace_back([=] { - try - { - results[i] = compiled_result{compile(*ctx, ins, preop, solution), ins}; - } - catch(const std::exception& e) - { - const auto trace_level = value_of(MIGRAPHX_TRACE_BENCHMARKING{}); - if(trace_level > 0) - std::cerr << "Exception in " + preop.name() + ": " + e.what() << std::endl; - results[i] = nullopt; - } - catch(...) - { - results[i] = nullopt; - } - }); - } - - template - void add_compiles(Vector& compiles) - { - if(config.has_value()) - { - const auto& problem = config->problem; - if(auto sol = ctx->get_problem_cache().get(preop.name(), problem)) - { - auto solution = sol.value(); - // No solution yet until benchmarked so skip for now - if(solution.is_null()) - return; - results.resize(1); - insert_compiles(compiles, solution, 0); - } - else - { - ctx->get_problem_cache().mark(preop.name(), problem); - const auto& solutions = config->solutions; - if(solutions.empty()) - MIGRAPHX_THROW("No solutions provided for " + preop.name() + " with " + - to_string(problem)); - results.resize(solutions.size()); - for(auto i : range(solutions.size())) - { - auto solution = solutions[i]; - insert_compiles(compiles, solution, i); - } - } - } - else - { - results.resize(1); - insert_compiles(compiles, value{}, 0); - } - } - std::string problem_string() const - { - if(config) - return to_string(config->problem); - return ""; - } - - const compiled_result& benchmark() const - { - const auto trace_level = value_of(MIGRAPHX_TRACE_BENCHMARKING{}); - if(trace_level > 0 and not results.empty()) - { - std::cout << "Benchmarking " << preop.name() << ": " << results.size() << " configs" - << std::endl; - } - if(results.empty()) - MIGRAPHX_THROW("No valid tuned compilation for " + preop.name() + " with " + - problem_string()); - if(results.size() == 1) - { - if(not results.front().has_value()) - MIGRAPHX_THROW("No valid tuned compilation for " + preop.name() + " with " + - problem_string()); - return *results.front(); - } - if(not config) - MIGRAPHX_THROW("Multiple kernels without config for " + preop.name()); - if(trace_level > 1) - std::cout << "Problem: " << config->problem << std::endl; - std::vector times; - times.reserve(results.size()); - std::transform(results.begin(), - results.end(), - config->solutions.begin(), - std::back_inserter(times), - [&](const auto& cr, const auto& solution) { - if(trace_level > 1) - std::cout << "Benchmarking solution: " << solution << std::endl; - if(not cr.has_value()) - { - if(trace_level > 1) - std::cout << "No binary" << std::endl; - return std::numeric_limits::max(); - } - if(trace_level > 2) - std::cout << *cr << std::endl; - /* - create a small program with insturction being compiled and call "replace" - on that which would insert all the compiled code objects, prefills etc. - necessary to run candidate code object - */ - program bench_prog; - auto* bench_mm = bench_prog.get_main_module(); - std::vector bench_ins_inputs; - - std::transform(cr->ins->inputs().begin(), - cr->ins->inputs().end(), - std::back_inserter(bench_ins_inputs), - [&](const auto& arg) { - return bench_mm->add_parameter( - std::to_string(bench_ins_inputs.size()), - arg->get_shape()); - }); - auto bench_ins = bench_mm->add_instruction( - cr->ins->get_operator(), bench_ins_inputs, cr->ins->module_inputs()); - cr->replace.replace(*bench_mm, bench_ins); - // do dead code elimination by directly removing instruction - bench_mm->remove_instruction(bench_ins); - auto t = time_program(*ctx, bench_prog, 20); - if(trace_level > 1) - std::cout << t << "ms" << std::endl; - return t; - }); - std::this_thread::sleep_for(std::chrono::milliseconds{50}); - auto i = std::distance(times.begin(), std::min_element(times.begin(), times.end())); - if(trace_level > 0) - std::cout << "Fastest solution: " << config->solutions.at(i) << std::endl; - ctx->get_problem_cache().insert(preop.name(), config->problem, config->solutions.at(i)); - if(not results[i].has_value()) - MIGRAPHX_THROW("No valid tuned compilation for " + preop.name() + " with " + - problem_string()); - auto skipped = std::count_if( - results.begin(), results.end(), [](const auto& cr) { return not cr.has_value(); }); - if(skipped > 0) - std::cout << "Skipped " << skipped << " configs for " << preop.name() << std::endl; - - return *results[i]; - } - - void replace(module& m) const - { - const auto& cr = benchmark(); - cr.replace.replace(m, cr.ins); - } -}; - -template -void par_compile(std::size_t n, F f) -{ - if(n == 0) - return; - auto d = value_of(MIGRAPHX_GPU_COMPILE_PARALLEL{}); - if(d == 0) - d = n; - par_for(n, n / d, f); -} - -struct compile_manager -{ - std::vector cps; - bool exhaustive = false; - - template - void add_plan(Ts&&... xs) - { - cps.push_back({std::forward(xs)...}); - } - - void update_configs() - { - par_compile(cps.size(), [&](auto i) { cps[i].update_config(exhaustive); }); - } - - void compile(module& m) - { - std::vector> compiles; - for(auto& cp : cps) - { - cp.add_compiles(compiles); - } - par_compile(compiles.size(), [&](auto i) { compiles[i](); }); - - // Replace and/or benchmark - for(const auto& cp : cps) - { - if(cp.results.empty()) - continue; - cp.replace(m); - } - - // Remove compile_plan already executed - cps.erase(std::remove_if(cps.begin(), - cps.end(), - [](const auto& cp) { return not cp.results.empty(); }), - cps.end()); - } -}; - -void compile_ops::apply(module& m) const -{ - compile_manager cm; - cm.exhaustive = exhaustive_tune; - // Find all precompile ops - for(auto ins : iterator_for(m)) - { - if(ins->name() != "gpu::precompile_op") - continue; - operation preop = any_cast(ins->get_operator()).op; - cm.add_plan(ctx, preop, ins); - } - cm.update_configs(); - cm.compile(m); - // Compile already tuned configs - cm.compile(m); - assert(cm.cps.empty()); -} - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compile_pointwise.cpp b/docker/rocm/migraphx/targets/gpu/compile_pointwise.cpp deleted file mode 100644 index ee682cf2c..000000000 --- a/docker/rocm/migraphx/targets/gpu/compile_pointwise.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -operation -compile_pointwise(context& ctx, const std::vector& in_shapes, const_module_ref pm) -{ - auto pf = gen::generate_pointwise(*pm, "inner_pointwise", true); - std::string lambda = "MIGRAPHX_LIFT(inner_pointwise)"; - auto kernel_name = gen::generate_name_from_ops(*pm, "kernel"); - return gpu::compile_op("pointwise", - ctx, - in_shapes, - {{"lambda", lambda}, {"preamble", pf}, {"kernel", kernel_name}}); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/compiler.cpp b/docker/rocm/migraphx/targets/gpu/compiler.cpp deleted file mode 100644 index 3b3b786e2..000000000 --- a/docker/rocm/migraphx/targets/gpu/compiler.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -namespace { -struct compiler_handle -{ - compiler_compile compile; - compiler_compile_op compile_op; - compiler_tuning_config get_tuning_config; -}; -} // namespace - -auto& compiler_map() -{ - static std::unordered_map m; // NOLINT - return m; -} - -void register_compiler(const std::string& name, - compiler_compile c, - compiler_compile_op cop, - compiler_tuning_config ctg) -{ - compiler_map()[name] = {std::move(c), std::move(cop), std::move(ctg)}; -} - -bool has_compiler_for(const std::string& name) { return compiler_map().count(name) > 0; } -compiler_replace -compile(context& ctx, instruction_ref ins, const operation& op, const value& solution) -{ - return compiler_map().at(op.name()).compile(ctx, ins, op, solution); -} -operation -compile_op(const std::string& name, context& ctx, const std::vector& inputs, const value& v) -{ - return compiler_map().at(name).compile_op(ctx, inputs, v); -} - -optional -get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool exhaustive) -{ - return compiler_map().at(op.name()).get_tuning_config(ctx, ins, op, exhaustive); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/argmax.cpp b/docker/rocm/migraphx/targets/gpu/device/argmax.cpp deleted file mode 100644 index e71a1b955..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/argmax.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void argmax(hipStream_t stream, - const argument& result, - const argument& arg, - int64_t axis, - bool select_last_index) -{ - if(select_last_index) - arg_op(argmax_op_last_index{}, stream, result, arg, axis); - else - arg_op(argmax_op_first_index{}, stream, result, arg, axis); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/argmin.cpp b/docker/rocm/migraphx/targets/gpu/device/argmin.cpp deleted file mode 100644 index 18338bc48..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/argmin.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void argmin(hipStream_t stream, - const argument& result, - const argument& arg, - int64_t axis, - bool select_last_index) -{ - if(select_last_index) - arg_op(argmin_op_last_index{}, stream, result, arg, axis); - else - arg_op(argmin_op_first_index{}, stream, result, arg, axis); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/contiguous.cpp b/docker/rocm/migraphx/targets/gpu/device/contiguous.cpp deleted file mode 100644 index 7d30aec54..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/contiguous.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void contiguous_nonstandard(hipStream_t stream, const argument& result, const argument& arg) -{ - shape s{result.get_shape().type(), result.get_shape().lens()}; - visit_all(result, arg)([&](auto output_v, auto input_v) { - hip_visit_views(output_v, input_v, s)([&](auto output, auto input, auto standard_shape) { - mi_gs_launch(stream, - standard_shape)([=](auto idx) __device__ { output[idx] = input[idx]; }); - }); - }); -} - -void contiguous_packed(hipStream_t stream, const argument& result, const argument& arg) -{ - index_int nelements = result.get_shape().elements(); - visit_all(result, arg)([&](auto output_v, auto input_v) { - const auto* input = device_cast(input_v.data()); - auto* output = device_cast(output_v.data()); - gs_launch(stream, nelements)([=](auto i) __device__ { output[i] = input[i]; }); - }); -} - -void contiguous(hipStream_t stream, const argument& result, const argument& arg) -{ - if(result.get_shape() == arg.get_shape() and result.get_shape().packed()) - contiguous_packed(stream, result, arg); - else - contiguous_nonstandard(stream, result, arg); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/fill.cpp b/docker/rocm/migraphx/targets/gpu/device/fill.cpp deleted file mode 100644 index ea6640b7e..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/fill.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void fill(hipStream_t stream, const argument& result, unsigned long val) -{ - nary(stream, result)([=]() __device__ { return val; }); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/array.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/array.hpp deleted file mode 100644 index 41d58e667..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/array.hpp +++ /dev/null @@ -1,185 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_ARRAY_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_ARRAY_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_ARRAY_OP(op, binary_op) \ - MIGRAPHX_DEVICE_CONSTEXPR hip_array& operator op(const hip_array& x) \ - { \ - for(index_int i = 0; i < N; i++) \ - d[i] op x[i]; \ - return *this; \ - } \ - MIGRAPHX_DEVICE_CONSTEXPR hip_array& operator op(const T& x) \ - { \ - for(index_int i = 0; i < N; i++) \ - d[i] op x; \ - return *this; \ - } \ - friend MIGRAPHX_DEVICE_CONSTEXPR hip_array operator binary_op(hip_array x, const hip_array& y) \ - { \ - return x op y; \ - } \ - friend MIGRAPHX_DEVICE_CONSTEXPR hip_array operator binary_op(hip_array x, const T& y) \ - { \ - return x op y; \ - } \ - friend MIGRAPHX_DEVICE_CONSTEXPR hip_array operator binary_op(const T& y, hip_array x) \ - { \ - return x op y; \ - } - -template -struct hip_array -{ - T d[N]; - MIGRAPHX_DEVICE_CONSTEXPR T& operator[](index_int i) { return d[i]; } - MIGRAPHX_DEVICE_CONSTEXPR const T& operator[](index_int i) const { return d[i]; } - - MIGRAPHX_DEVICE_CONSTEXPR T& front() { return d[0]; } - MIGRAPHX_DEVICE_CONSTEXPR const T& front() const { return d[0]; } - - MIGRAPHX_DEVICE_CONSTEXPR T& back() { return d[N - 1]; } - MIGRAPHX_DEVICE_CONSTEXPR const T& back() const { return d[N - 1]; } - - MIGRAPHX_DEVICE_CONSTEXPR T* data() { return d; } - MIGRAPHX_DEVICE_CONSTEXPR const T* data() const { return d; } - - MIGRAPHX_DEVICE_CONSTEXPR std::integral_constant size() const { return {}; } - - MIGRAPHX_DEVICE_CONSTEXPR T* begin() { return d; } - MIGRAPHX_DEVICE_CONSTEXPR const T* begin() const { return d; } - - MIGRAPHX_DEVICE_CONSTEXPR T* end() { return d + size(); } - MIGRAPHX_DEVICE_CONSTEXPR const T* end() const { return d + size(); } - - MIGRAPHX_DEVICE_CONSTEXPR T dot(const hip_array& x) const - { - T result = 0; - for(index_int i = 0; i < N; i++) - result += x[i] * d[i]; - return result; - } - - MIGRAPHX_DEVICE_CONSTEXPR T product() const - { - T result = 1; - for(index_int i = 0; i < N; i++) - result *= d[i]; - return result; - } - - MIGRAPHX_DEVICE_CONSTEXPR T single(index_int width = 100) const - { - T result = 0; - T a = 1; - for(index_int i = 0; i < N; i++) - { - result += d[N - i - 1] * a; - a *= width; - } - return result; - } - - MIGRAPHX_DEVICE_ARRAY_OP(+=, +) - MIGRAPHX_DEVICE_ARRAY_OP(*=, *) - MIGRAPHX_DEVICE_ARRAY_OP(/=, /) - MIGRAPHX_DEVICE_ARRAY_OP(%=, %) - MIGRAPHX_DEVICE_ARRAY_OP(&=, &) - MIGRAPHX_DEVICE_ARRAY_OP(|=, |) - MIGRAPHX_DEVICE_ARRAY_OP(^=, ^) - - friend MIGRAPHX_DEVICE_CONSTEXPR bool operator==(const hip_array& x, const hip_array& y) - { - for(index_int i = 0; i < N; i++) - { - if(x[i] != y[i]) - return false; - } - return true; - } - - friend MIGRAPHX_DEVICE_CONSTEXPR bool operator!=(const hip_array& x, const hip_array& y) - { - return not(x == y); - } - // This uses the product order rather than lexical order - friend MIGRAPHX_DEVICE_CONSTEXPR bool operator<(const hip_array& x, const hip_array& y) - { - for(index_int i = 0; i < N; i++) - { - if(not(x[i] < y[i])) - return false; - } - return true; - } - friend MIGRAPHX_DEVICE_CONSTEXPR bool operator>(const hip_array& x, const hip_array& y) - { - return y < x; - } - friend MIGRAPHX_DEVICE_CONSTEXPR bool operator<=(const hip_array& x, const hip_array& y) - { - return (x < y) or (x == y); - } - friend MIGRAPHX_DEVICE_CONSTEXPR bool operator>=(const hip_array& x, const hip_array& y) - { - return (y < x) or (x == y); - } - - MIGRAPHX_DEVICE_CONSTEXPR hip_array carry(hip_array result) const - { - uint32_t overflow = 0; - for(std::ptrdiff_t i = result.size() - 1; i > 0; i--) - { - auto z = result[i] + overflow; - // Reset overflow - overflow = 0; - // Compute overflow using while loop instead of mod - while(z >= d[i]) - { - z -= d[i]; - overflow += 1; - } - result[i] = z; - } - result[0] += overflow; - return result; - } -}; - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/fast_div.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/fast_div.hpp deleted file mode 100644 index 70c355135..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/fast_div.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_FAST_DIV_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_FAST_DIV_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -constexpr const uint64_t fast_div_shift = 42; -inline uint64_t encode_divisor(uint64_t divisor) -{ - if(divisor == 0) - return 0; - auto p = uint64_t{1} << fast_div_shift; - return (p + divisor - 1) / divisor; -} - -inline constexpr bool is_divisor_encodable(uint64_t i) -{ - return i < (uint64_t{1} << (fast_div_shift / 2)); -} - -MIGRAPHX_DEVICE_CONSTEXPR uint64_t fast_div(uint64_t dividend, uint64_t encoded_divisor) -{ - return (dividend * encoded_divisor) >> fast_div_shift; -} - -MIGRAPHX_DEVICE_CONSTEXPR uint64_t remainder(uint64_t result, uint64_t dividend, uint64_t divisor) -{ - return dividend - divisor * result; -} - -MIGRAPHX_DEVICE_CONSTEXPR uint64_t fast_mod(uint64_t dividend, - uint64_t divisor, - uint64_t encoded_divisor) -{ - return remainder(fast_div(dividend, encoded_divisor), dividend, divisor); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/float_equal.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/float_equal.hpp deleted file mode 100644 index a5f18fc5a..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/float_equal.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_DEVICE_FLOAT_EQUAL_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_DEVICE_FLOAT_EQUAL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -using common_type = typename std::common_type::type; - -template {})> -__device__ bool float_equal_device(T x, T y) -{ - return std::isfinite(x) and std::isfinite(y) and - std::nextafter(x, std::numeric_limits::lowest()) <= y and - std::nextafter(x, std::numeric_limits::max()) >= y; -} - -template <> -__device__ bool float_equal_device(__bf16 x, __bf16 y) // NOLINT(misc-definitions-in-headers) -{ - float xf = x; - float yf = y; - return std::isfinite(xf) and std::isfinite(yf) and - std::nextafter(xf, std::numeric_limits::lowest()) <= yf and - std::nextafter(xf, std::numeric_limits::max()) >= yf; -} - -template {})> -__device__ bool float_equal_device(T x, T y) -{ - return x == y; -} - -template -__device__ bool float_equal(T x, U y) -{ - return float_equal_device>(x, y); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/launch.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/launch.hpp deleted file mode 100644 index 573f57b3b..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/launch.hpp +++ /dev/null @@ -1,146 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_LAUNCH_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_LAUNCH_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -struct index -{ - index_int global = 0; - index_int local = 0; - index_int group = 0; - - __device__ index_int nglobal() const { return blockDim.x * gridDim.x; } // NOLINT - - __device__ index_int nlocal() const { return blockDim.x; } // NOLINT - - template - __device__ void global_stride(index_int n, F f) const - { - const auto stride = nglobal(); - for(index_int i = global; i < n; i += stride) - { - f(i); - } - } - - template - __device__ void local_stride(index_int n, F f) const - { - const auto stride = nlocal(); - for(index_int i = local; i < n; i += stride) - { - f(i); - } - } -}; - -template -__global__ void launcher(F f) -{ - index idx{blockIdx.x * blockDim.x + threadIdx.x, threadIdx.x, blockIdx.x}; // NOLINT - f(idx); -} - -inline auto launch(hipStream_t stream, index_int global, index_int local) -{ - return [=](auto f) { - assert(local > 0); - assert(global > 0); - using f_type = decltype(f); - dim3 nblocks(global / local); - dim3 nthreads(local); - /* - hipGetLastError() returns error for the first failed HIP call that happened previously. - MIGraphX calls into various backend libraries and failed HIP calls can also happen there. - Calling hipGetLastError() would reset error code to hipSuccess, so that inside MIGraphX - failed call to hipLaunchKernelGGL() can be captured. - */ - hipError_t flush_call = hipGetLastError(); - (void)(flush_call); - // cppcheck-suppress migraphx-UseDeviceLaunch - hipLaunchKernelGGL((launcher), nblocks, nthreads, 0, stream, f); - hipError_t kernel_launch_status = hipGetLastError(); - if(kernel_launch_status != hipSuccess) - { - std::string message = hipGetErrorString(kernel_launch_status); - if(not contains(get_targets(), get_device_name())) - { - message += ". Trying to run a kernel for " + get_device_name() + - " but MIGraphX was built for targets " + get_targets_as_string() + - ". Please rebuild MIGraphX with -DGPU_TARGETS='" + get_device_name() + - "'."; - } - MIGRAPHX_THROW("MIGraphX device kernel failed to launch with error: " + message); - } - }; -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto gs_invoke(F&& f, index_int i, index idx) -> decltype(f(i, idx)) -{ - return f(i, idx); -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto gs_invoke(F&& f, index_int i, index) -> decltype(f(i)) -{ - return f(i); -} - -inline auto gs_launch(hipStream_t stream, index_int n, index_int local = 1024) -{ - index_int groups = (n + local - 1) / local; - // max possible number of blocks is set to 1B (1,073,741,824) - index_int nglobal = std::min(1073741824, groups) * local; - - return [=](auto f) { - launch(stream, nglobal, local)([=](auto idx) __device__ { - idx.global_stride(n, [&](auto i) { gs_invoke(f, i, idx); }); - }); - }; -} - -#ifdef MIGRAPHX_USE_CLANG_TIDY -#define MIGRAPHX_DEVICE_SHARED -#else -#define MIGRAPHX_DEVICE_SHARED __shared__ -#endif - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/multi_index.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/multi_index.hpp deleted file mode 100644 index 6be513a88..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/multi_index.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MULTI_INDEX_HPP -#define MIGRAPHX_GUARD_RTGLIB_MULTI_INDEX_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -struct multi_index -{ - using hip_index = hip_array; - hip_index id{}; - hip_index stride{}; - - MIGRAPHX_DEVICE_CONSTEXPR auto for_stride(hip_index n) const - { - // f should return void, but this helps with type deduction - return [=](auto f) -> decltype(f(hip_index{})) { - for(hip_index i = id; i < n; i = n.carry(i + stride)) - { - f(i); - } - }; - } -}; - -template -__device__ __host__ auto deduce_for_stride(ForStride fs) -> decltype(fs(id{})); - -MIGRAPHX_DEVICE_CONSTEXPR multi_index<1> make_multi_index(index_int i, index_int n) -{ - return {{i}, {n}}; -} - -template -MIGRAPHX_DEVICE_CONSTEXPR multi_index -make_multi_index(const hip_shape& s, index_int i, index_int n) -{ - return {s.multi(i), s.multi(n)}; -} - -template -MIGRAPHX_DEVICE_CONSTEXPR multi_index -make_multi_index(const hip_shape& s, index_int i, const hip_array& n) -{ - return {s.multi(i), n}; -} - -template -inline auto mi_nglobal(const hip_shape& s, index_int nlocal) -{ - assert(s.standard); - assert(s.elements() > 0); - index_int n = s.elements(); - index_int groups = (n + nlocal - 1) / nlocal; - // max possible number of blocks is set to 1B (1,073,741,824) - index_int nglobal = std::min(1073741824, groups) * nlocal; - - assert(groups > 0); - assert(nglobal > 0); - auto nglobal_multi = s.multi(nglobal); - - // Skip checking this, since this will cause metadata to not be generated - // for some unknown reason. - // - // assert(std::any_of(nglobal_multi.begin(), nglobal_multi.end(), [](auto x){return x>0;})); - - // cppcheck-suppress migraphx-RedundantLocalVariable - return nglobal_multi; -} - -template -inline auto mi_nlocal(const hip_shape& s, index_int local) -{ - assert(s.standard); - assert(s.elements() > 0); - auto nlocal_multi = s.multi(local); - - // Skip checking this, since this will cause metadata to not be generated - // for some unknown reason. - // - // assert(std::any_of(nlocal_multi.begin(), nlocal_multi.end(), [](auto x){return x>0;})); - - // cppcheck-suppress migraphx-RedundantLocalVariable - return nlocal_multi; -} - -template -inline auto mi_launch(hipStream_t stream, const hip_shape& global, index_int nlocal = 1024) -{ - auto nglobal_multi = mi_nglobal(global, nlocal); - auto nglobal = global.index(nglobal_multi); - - return [=](auto f) { - launch(stream, nglobal, nlocal)([=](auto idx) __device__ { - auto midx = make_multi_index(global, idx.global, nglobal_multi); - f(idx, midx.for_stride(global.lens)); - }); - }; -} - -template -inline auto mi_launch(hipStream_t stream, - const hip_shape& global, - const hip_shape& local, - index_int nlocal = 1024) -{ - auto nglobal_multi = mi_nglobal(global, 1); - auto nglobal = global.index(nglobal_multi); - auto nlocal_multi = mi_nlocal(local, nlocal); - - return [=](auto f) { - launch(stream, nglobal * nlocal, nlocal)([=](auto idx) { - // TODO: Use fast div for nlocal - auto midx = make_multi_index(global, idx.global / nlocal, nglobal_multi); - auto lidx = make_multi_index(local, idx.local, nlocal_multi); - f(idx, midx.for_stride(global.lens), lidx.for_stride(local.lens)); - }); - }; -} - -template -inline auto mi_gs_launch(hipStream_t stream, const hip_shape& global, index_int nlocal = 1024) -{ - return [=](auto f) { - mi_launch(stream, global, nlocal)([=](auto, auto g) { g([&](auto i) { f(i); }); }); - }; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/nary.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/nary.hpp deleted file mode 100644 index e9af38473..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/nary.hpp +++ /dev/null @@ -1,473 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_NARY_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_NARY_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_NARY); - -// NOLINTNEXTLINE -#define MIGRAPHX_TRACE_NARY_FUNCTION \ - if(enabled(MIGRAPHX_TRACE_NARY{})) \ - std::cout << "nary device function: " << __PRETTY_FUNCTION__ << std::endl; - -template -constexpr auto pack(Ts... xs) -{ - return [=](auto f) { return f(xs...); }; -} - -template -auto nary_nonstandard_nonpacked_impl(hipStream_t stream, F f, argument result, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - shape s{result.get_shape().type(), result.get_shape().lens()}; - hip_visit_all(s, result, args...)([&](auto standard_shape, auto output, auto... inputs) { - mi_gs_launch(stream, - standard_shape)([=](auto idx) __device__ { output[idx] = f(inputs[idx]...); }); - }); -} - -inline auto create_broadcast_index(index_int len, index_int stride) -{ - auto next_stride = stride * len; - auto e_next_stride = encode_divisor(next_stride); - auto e_stride = encode_divisor(stride); - return [=](auto i) __device__ { - // ( i % next_stride) / stride - return fast_div(i, e_stride) - len * fast_div(i, e_next_stride); - }; -} - -template -auto nary_nonstandard_packed_impl(hipStream_t stream, - F f, - const argument& result, - Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - auto arg_shape = make_array(args...).front().get_shape(); - auto perm = find_permutation(arg_shape); - auto s = reorder_shape(arg_shape, perm); - hip_visit_all(s, result.reshape(reorder_shape(result.get_shape(), perm)), args.reshape(s)...)( - [&](auto standard_shape, auto output, auto... inputs) { - mi_gs_launch(stream, standard_shape)( - [=](auto idx) __device__ { output[idx] = f(inputs[idx]...); }); - }); -} - -template -void nary_broadcast_vec_impl( - hipStream_t stream, F f, argument result, argument barg, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - const auto& output_shape = result.get_shape(); - const auto& b_shape = barg.get_shape(); - auto bdim = - std::distance(b_shape.strides().begin(), - std::find_if(b_shape.strides().begin(), b_shape.strides().end(), [](auto x) { - return x != 0; - })); - auto bdim_len = output_shape.lens()[bdim]; - auto bdim_stride = output_shape.strides()[bdim]; - auto broadcast_idx = create_broadcast_index(bdim_len, bdim_stride); - - const index_int vec_size = 4; - const index_int nlocal = 1024; - const index_int nglobal = 256 * nlocal; - const index_int bdim_vec_len = bdim_len / vec_size; - hip_vec_visit_all(result, barg, args...)( - [&](auto output, auto binput, auto... inputs) { - using type = typename decltype(output)::value_type; - const index_int nelements = output.size() / vec_size; - launch(stream, nglobal, nlocal)([=](auto idx) __device__ { - MIGRAPHX_DEVICE_SHARED type buffer[2048 / vec_size]; - // Load bias into LDS - for(size_t i = idx.local; i < bdim_vec_len; i += nlocal) - { - buffer[i] = binput.data()[i]; - } - __syncthreads(); - const auto* bp = as_pointer(buffer); - // Process the data - for(size_t i = idx.global; i < nelements; i += nglobal) - { - auto bidx = broadcast_idx(i * vec_size); - auto b = bp[bidx]; - auto out = output.data()[i]; - for(index_int j = 0; j < vec_size; j++) - { - out[j] = f(inputs.data()[i][j]..., b); - } - output.data()[i] = out; - } - }); - }); -} - -template -void nary_broadcast_impl(hipStream_t stream, F f, argument result, argument barg, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - const auto& output_shape = result.get_shape(); - const auto& b_shape = barg.get_shape(); - auto bdim = - std::distance(b_shape.strides().begin(), - std::find_if(b_shape.strides().begin(), b_shape.strides().end(), [](auto x) { - return x != 0; - })); - auto bdim_len = output_shape.lens()[bdim]; - auto bdim_stride = output_shape.strides()[bdim]; - auto broadcast_idx = create_broadcast_index(bdim_len, bdim_stride); - - const index_int nlocal = 1024; - const index_int nglobal = 256 * nlocal; - index_int nelements = result.get_shape().elements(); - hip_visit_all(result, barg, args...)([&](auto output, auto binput, auto... inputs) { - using type = typename decltype(output)::value_type; - launch(stream, nglobal, nlocal)([=](auto idx) __device__ { - MIGRAPHX_DEVICE_SHARED type buffer[2048]; - // Load bias into LDS - for(size_t i = idx.local; i < bdim_len; i += nlocal) - { - buffer[i] = binput.data()[i]; - } - __syncthreads(); - // Process the data - for(size_t i = idx.global; i < nelements; i += nglobal) - { - auto bidx = broadcast_idx(i); - auto b = buffer[bidx]; - output.data()[i] = f(inputs.data()[i]..., b); - } - }); - }); -} - -template -void nary_double_broadcast_vec_impl( - hipStream_t stream, F f, argument result, argument barg1, argument barg2, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - assert(barg1.get_shape().broadcasted()); - assert(barg2.get_shape().broadcasted()); - assert(barg1.get_shape() == barg2.get_shape()); - const auto& output_shape = result.get_shape(); - const auto& b_shape = barg1.get_shape(); - auto bdim = - std::distance(b_shape.strides().begin(), - std::find_if(b_shape.strides().begin(), b_shape.strides().end(), [](auto x) { - return x != 0; - })); - auto bdim_len = output_shape.lens()[bdim]; - auto bdim_stride = output_shape.strides()[bdim]; - auto broadcast_idx = create_broadcast_index(bdim_len, bdim_stride); - - const index_int vec_size = 4; - const index_int nlocal = 1024; - const index_int nglobal = 256 * nlocal; - const index_int bdim_vec_len = bdim_len / vec_size; - hip_vec_visit_all(result, barg1, barg2, args...)( - [&](auto output, auto binput1, auto binput2, auto... inputs) { - using type = typename decltype(output)::value_type; - const index_int nelements = output.size() / vec_size; - launch(stream, nglobal, nlocal)([=](auto idx) __device__ { - MIGRAPHX_DEVICE_SHARED type buffer[2048 / vec_size]; - // Load bias into LDS - for(size_t i = idx.local; i < bdim_vec_len; i += nlocal) - { - buffer[i] = binput1.data()[i]; - } - for(size_t i = idx.local; i < bdim_vec_len; i += nlocal) - { - buffer[i + bdim_vec_len] = binput2.data()[i]; - } - __syncthreads(); - const auto* bp = as_pointer(buffer); - // Process the data - for(size_t i = idx.global; i < nelements; i += nglobal) - { - auto bidx = broadcast_idx(i * vec_size); - auto b1 = bp[bidx]; - auto b2 = bp[bidx + bdim_len]; - auto out = output.data()[i]; - for(index_int j = 0; j < vec_size; j++) - { - out[j] = f(inputs.data()[i][j]..., b2, b1); - } - output.data()[i] = out; - } - }); - }); -} - -template -void nary_double_broadcast_impl( - hipStream_t stream, F f, argument result, argument barg1, argument barg2, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - assert(barg1.get_shape().broadcasted()); - assert(barg2.get_shape().broadcasted()); - assert(barg1.get_shape() == barg2.get_shape()); - const auto& output_shape = result.get_shape(); - const auto& b_shape = barg1.get_shape(); - auto bdim = - std::distance(b_shape.strides().begin(), - std::find_if(b_shape.strides().begin(), b_shape.strides().end(), [](auto x) { - return x != 0; - })); - auto bdim_len = output_shape.lens()[bdim]; - auto bdim_stride = output_shape.strides()[bdim]; - auto broadcast_idx = create_broadcast_index(bdim_len, bdim_stride); - - const index_int nlocal = 1024; - const index_int nglobal = 256 * nlocal; - index_int nelements = result.get_shape().elements(); - hip_visit_all(result, barg1, barg2, args...)( - [&](auto output, auto binput1, auto binput2, auto... inputs) { - using type = typename decltype(output)::value_type; - launch(stream, nglobal, nlocal)([=](auto idx) __device__ { - MIGRAPHX_DEVICE_SHARED type buffer[2048]; - // Load bias into LDS - for(size_t i = idx.local; i < bdim_len; i += nlocal) - { - buffer[i] = binput1.data()[i]; - } - for(size_t i = idx.local; i < bdim_len; i += nlocal) - { - buffer[i + bdim_len] = binput2.data()[i]; - } - __syncthreads(); - // Process the data - for(size_t i = idx.global; i < nelements; i += nglobal) - { - auto bidx = broadcast_idx(i); - auto b1 = buffer[bidx]; - auto b2 = buffer[bidx + bdim_len]; - output.data()[i] = f(inputs.data()[i]..., b2, b1); - } - }); - }); -} - -template -void nary_standard_vec_impl(hipStream_t stream, F f, argument result, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - const auto& output_shape = result.get_shape(); - visit_all(result, args...)([&](auto output, auto... inputs) { - using type = device_type>; - const index_int vec_size = 4; - auto data = pack_vec<4>(device_cast(inputs.data())...); - auto* outp = as_vec<4>(device_cast(output.data())); - gs_launch(stream, output_shape.elements() / vec_size)([=](auto i) __device__ { - vec out = outp[i]; - data( - [&](auto... xs) { - for(index_int j = 0; j < vec_size; j++) - { - out[j] = f(xs[j]...); - } - }, - i); - outp[i] = out; - }); - }); -} - -template -void nary_standard_impl(hipStream_t stream, F f, argument result, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - index_int nelements = result.get_shape().elements(); - hip_pointer_visit_all(result, args...)([&](auto output, auto... inputs) { - gs_launch(stream, nelements)([=](auto i) __device__ { output[i] = f(inputs[i]...); }); - }); -} - -template -void nary_impl(hipStream_t stream, F f, argument result, Arguments... args) -{ - MIGRAPHX_TRACE_NARY_FUNCTION - const auto shapes = make_array(args.get_shape()...); - const bool standard = all_of(shapes, [](const shape& s) { return s.standard(); }); - const bool packed = - all_of(shapes, [](const shape& s) { return s.packed() and not s.broadcasted(); }); - const bool same_shapes = - all_of(shapes, [&](const shape& s) { return s == result.get_shape(); }); - const bool same_input_shapes = all_of(shapes, [&](const shape& s) { return s == shapes[0]; }); - if((result.get_shape().standard() and standard) or (packed and same_shapes)) - nary_standard_impl(stream, f, result, args...); - else if(packed and same_input_shapes) - nary_nonstandard_packed_impl(stream, f, result, args...); - else - nary_nonstandard_nonpacked_impl(stream, f, result, args...); -} - -template -auto nary_nonstandard(hipStream_t stream, argument result, Arguments... args) -{ - return [=](auto f) { nary_nonstandard_nonpacked_impl(stream, f, result, args...); }; -} - -template -auto nary_standard(hipStream_t stream, argument result, Arguments... args) -{ - return [=](auto f) { nary_standard_impl(stream, f, result, args...); }; -} - -template -bool broadcastable(bool& divisible_by_4, - index_int max_size, - const argument& result, - const argument& barg, - const Arguments&... args) -{ - divisible_by_4 = false; - auto bshape = barg.get_shape(); - const bool standard = - all_of({args.get_shape()...}, [](const shape& s) { return s.standard(); }); - const bool same_shapes = - all_of({args.get_shape()...}, [&](const shape& s) { return s == result.get_shape(); }); - // TODO: Check result and args shape is the same - if(standard and same_shapes and bshape.broadcasted() and not bshape.scalar()) - { - auto not_zero = [](auto x) { return x != 0; }; - const auto& strides = bshape.strides(); - auto b_it = std::find_if(strides.begin(), strides.end(), not_zero); - auto b_idx = std::distance(strides.begin(), b_it); - auto b_len = result.get_shape().lens()[b_idx]; - auto b_stride = result.get_shape().strides()[b_idx]; - assert(bshape.lens()[b_idx] == b_len); - if(b_len <= max_size and std::none_of(std::next(b_it), strides.end(), not_zero) and - is_divisor_encodable(b_stride * b_len)) - { - - divisible_by_4 = (b_len % 4 == 0) and (b_stride % 4 == 0) and - (front_args(args...).get_shape().elements() % 4 == 0); - return true; - } - } - return false; -} - -inline bool broadcastable(bool& divisible_by_4, index_int, const argument&, const argument&) -{ - divisible_by_4 = false; - return false; -} - -// Nullary -inline auto nary(hipStream_t stream, argument result) -{ - return [=](auto f) { nary_standard_impl(stream, f, result); }; -} - -// Unary -inline auto nary(hipStream_t stream, argument result, argument arg) -{ - return [=](auto f) { nary_impl(stream, f, result, arg); }; -} - -// Binary -inline auto nary(hipStream_t stream, argument result, argument arg, argument barg) -{ - return [=](auto f) { - bool divisible_by_4 = false; - if(broadcastable(divisible_by_4, 2048, result, barg, arg)) - { - if(divisible_by_4) - nary_broadcast_vec_impl(stream, f, result, barg, arg); - else - nary_broadcast_impl(stream, f, result, barg, arg); - } - else - { - nary_impl(stream, f, result, arg, barg); - } - }; -} - -template -auto nary(hipStream_t stream, argument result, Arguments... args) -{ - static_assert(sizeof...(args) > 2, "Args needs to be greater than 2"); - return [=](auto f) { - auto barg1 = back_args(args...); - bool fallback1 = pop_back_args(args...)([&](auto&&... args2) { - auto barg2 = back_args(args2...); - bool fallback2 = - barg2.get_shape() != barg1.get_shape() or not barg2.get_shape().broadcasted() or - pop_back_args(args2...)([&](auto&&... args3) { - bool divisible_by_4 = false; - if(broadcastable(divisible_by_4, 1024, result, barg2, args3...)) - { - if(divisible_by_4) - nary_double_broadcast_vec_impl( - stream, f, result, barg1, barg2, args3...); - else - nary_double_broadcast_impl(stream, f, result, barg1, barg2, args3...); - return false; - } - return true; - }); - if(not fallback2) - return false; - bool divisible_by_4 = false; - if(broadcastable(divisible_by_4, 2048, result, barg1, args2...)) - { - if(divisible_by_4) - nary_broadcast_vec_impl(stream, f, result, barg1, args2...); - else - nary_broadcast_impl(stream, f, result, barg1, args2...); - return false; - } - return true; - }); - if(fallback1) - nary_impl(stream, f, result, args...); - }; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/reduce.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/reduce.hpp deleted file mode 100644 index ae796c66e..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/reduce.hpp +++ /dev/null @@ -1,311 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_REDUCE_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_REDUCE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -#ifdef MIGRAPHX_NO_DPP - -template {})> -__device__ auto block_reduce(index idx, Op op, T init, ForStride fs, F f) -{ - using type = decltype(f(deduce_for_stride(fs))); - MIGRAPHX_DEVICE_SHARED type buffer[N]; - type x = init; - fs([&](auto i) { x = op(x, f(i)); }); - buffer[idx.local] = x; - __syncthreads(); - - for(index_int s = 1; s < idx.nlocal(); s *= 2) - { - const index_int index = 2 * s * idx.local; - if(index + s < idx.nlocal()) - { - buffer[index] = op(buffer[index], buffer[index + s]); - } - __syncthreads(); - } - return buffer[0]; -} - -#else -constexpr unsigned int dpp_row_shr(unsigned int x) { return 0x110u | x; } - -constexpr unsigned int dpp_row_bcast(unsigned int x) -{ - unsigned int y = 0; - switch(x) - { - case 15: y = 0x142; break; - case 31: y = 0x143; break; - default: throw std::runtime_error("Unknown bcast"); - } - return y; -} - -template -__device__ T dpp_mov(T& x) -{ - static const index_int n = sizeof(T) < 4 ? 1 : sizeof(T) / 4; - union type - { - uint32_t reg[n]; - T data; - }; - type output{}; - type input{}; - // cppcheck-suppress unreadVariable - input.data = x; - for(index_int i = 0; i < n; i++) - { - output.reg[i] = __hip_move_dpp(input.reg[i], DppCtrl, RowMask, BankMask, BoundCtrl); - } - return output.data; -} - -template -__device__ void dpp_reduce(T& in, Op op) -{ - T out{}; - out = dpp_mov(in); - in = op(in, out); - out = dpp_mov(in); - in = op(in, out); - out = dpp_mov(in); - in = op(in, out); - out = dpp_mov(in); - in = op(in, out); -#if __AMDGCN_WAVEFRONT_SIZE == 64 - out = dpp_mov(in); - in = op(in, out); - out = dpp_mov(in); - in = op(in, out); -#endif -} - -__device__ inline void dpp_reduce(float& x, sum) -{ -#if defined(MIGRAPHX_USE_CLANG_TIDY) || defined(CPPCHECK) - x = 1; -#else - __asm__ volatile("s_nop 4\n" - "v_add_f32 %0 %0 %0 row_shr:1\n" - "s_nop 1\n" - "v_add_f32 %0 %0 %0 row_shr:2\n" - "s_nop 1\n" - "v_add_f32 %0 %0 %0 row_shr:4 bank_mask:0xe\n" - "s_nop 1\n" - "v_add_f32 %0 %0 %0 row_shr:8 bank_mask:0xc\n" - "s_nop 1\n" -#if __AMDGCN_WAVEFRONT_SIZE == 64 - "v_add_f32 %0 %0 %0 row_bcast:15 row_mask:0xa\n" - "s_nop 1\n" - "v_add_f32 %0 %0 %0 row_bcast:31 row_mask:0xc\n" -#endif - "s_nop 1\n" - : "=v"(x) - : "0"(x)); -#endif -} - -template {})> -__device__ auto block_reduce(index idx, Op op, T init, ForStride fs, F f) -{ - -#if __AMDGCN_WAVEFRONT_SIZE == 32 - constexpr index_int nthreads = 16; -#else - constexpr index_int nthreads = 64; -#endif - using type = decltype(f(deduce_for_stride(fs))); - MIGRAPHX_DEVICE_SHARED type buffer[N / nthreads]; - type x = init; - fs([&](auto i) { x = op(x, f(i)); }); - dpp_reduce(x, op); - - const auto ldsidx = idx.local / nthreads; - if((idx.local % nthreads) == nthreads - 1) - { - buffer[ldsidx] = x; - } - __syncthreads(); - - type y = init; - for(index_int i = 0; i < idx.nlocal() / nthreads; i++) - { - y = op(y, buffer[i]); - } - return y; -} -#endif -template -__device__ auto block_reduce(index idx, Op op, T init, index_int n, F f) -{ - auto midx = make_multi_index(idx.local, idx.nlocal()); - // Workaround hcc, create a local array - auto fs = midx.id; - fs[0] = n; - return block_reduce( - idx, op, init, midx.for_stride(fs), [&](auto mi) __device__ { return f(mi[0]); }); -} -constexpr index_int compute_block_size(index_int n, index_int max_block_size) -{ - size_t block_size = 64; - while(block_size < max_block_size and block_size < n) - block_size *= 2; - return block_size; -} - -inline std::vector get_reduce_lens(const std::vector& input_lens, - const std::vector& output_lens) -{ - std::vector reduce_lens; - std::transform(output_lens.begin(), - output_lens.end(), - input_lens.begin(), - std::back_inserter(reduce_lens), - [](auto x, auto y) -> index_int { - if(x == y) - return 1; - else - return y; - }); - return reduce_lens; -} - -template -void reduce_multi_impl(hipStream_t stream, - const argument& result, - const argument& arg, - Op op, - T init, - Input read_input, - Output read_output, - const shape& reduce_slice) -{ - hip_visit_all(result, arg, reduce_slice)([&](auto output, auto input, auto reduce_shape) { - auto relements = reduce_slice.elements(); - - const index_int max_block_size = 256; - const index_int block_size = compute_block_size(relements, max_block_size); - mi_launch(stream, output.get_shape(), reduce_shape, block_size)( - [=](auto idx, auto global, auto local) __device__ { - global([&](auto i) __device__ { - auto r = - block_reduce(idx, op, init, local, [&](auto j) __device__ { - return read_input(input[i + j]); - }); - if(idx.local == 0) - output[i] = read_output(r); - }); - }); - }); -} - -template -void reduce_standard_impl(hipStream_t stream, - const argument& result, - const argument& arg, - Op op, - T init, - Input read_input, - Output read_output, - index_int relements) -{ - hip_visit_all(result, arg)([&](auto output, auto input) { - auto nelements = result.get_shape().elements(); - - const index_int max_block_size = 256; - const index_int block_size = compute_block_size(relements, max_block_size); - gs_launch(stream, nelements * block_size, block_size)([=](auto i, auto idx) __device__ { - const auto out_idx = i / block_size; - const auto base_idx = out_idx * relements; - auto r = block_reduce(idx, op, init, relements, [&](auto j) __device__ { - return read_input(input.data()[base_idx + j]); - }); - if(idx.local == 0) - output.data()[out_idx] = read_output(r); - }); - }); -} - -template -void reduce(hipStream_t stream, - const argument& result, - const argument& arg, - Op op, - T init, - Input read_input, - Output read_output) -{ - auto&& output_shape = result.get_shape(); - auto&& input_shape = arg.get_shape(); - auto input_lens = input_shape.lens(); - auto output_lens = output_shape.lens(); - assert(output_lens.size() == input_lens.size()); - if(input_shape.standard() and output_shape.standard() and - output_lens.back() != input_lens.back() and - std::equal(output_lens.begin(), std::prev(output_lens.end()), input_lens.begin())) - { - reduce_standard_impl( - stream, result, arg, op, init, read_input, read_output, input_lens.back()); - } - else - { - std::vector reduce_lens = get_reduce_lens(input_lens, output_lens); - shape reduce_slice{output_shape.type(), reduce_lens}; - reduce_multi_impl(stream, result, arg, op, init, read_input, read_output, reduce_slice); - } -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_NO_DPP diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/reduce_ops.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/reduce_ops.hpp deleted file mode 100644 index 6bafb0d08..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/reduce_ops.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_DEVICE_REDUCE_OPS_HPP -#define MIGRAPHX_GUARD_DEVICE_REDUCE_OPS_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -struct sum -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return x + y; - } -}; - -struct product -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return x * y; - } -}; - -struct id -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x) const - { - return x; - } -}; - -struct mean -{ - size_t item_num = 1; - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x) const - { - return x / static_cast(item_num); - } -}; - -struct max -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return (x > y) ? x : y; - } -}; - -struct min -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return (x < y) ? x : y; - } -}; - -struct lowest -{ - template - __device__ __host__ operator T() const - { - return device_cast(std::numeric_limits>::lowest()); - } -}; - -struct highest -{ - template - __device__ __host__ operator T() const - { - return device_cast(std::numeric_limits>::max()); - } -}; - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_DEVICE_REDUCE_OPS_HPP diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/scan.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/scan.hpp deleted file mode 100644 index 5a66f7f73..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/scan.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_DEVICE_SCAN_HPP -#define MIGRAPHX_GUARD_DEVICE_SCAN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template {})> -__device__ void block_scan(index idx, Op op, T init, ForStride fs, Input input, Output output) -{ - using type = decltype(input(deduce_for_stride(fs))); - MIGRAPHX_DEVICE_SHARED type buffer[2][N]; - type x = init; - fs([&](auto i) { - index_int iout = 0; - index_int iin = 1; - if(idx.local == 0) - buffer[iout][idx.local] = op(input(i), x); - else - buffer[iout][idx.local] = input(i); - __syncthreads(); - for(index_int s = 1; s < idx.nlocal(); s *= 2) - { - iout = 1 - iout; - iin = 1 - iin; - if(idx.local >= s) - { - buffer[iout][idx.local] = op(buffer[iin][idx.local], buffer[iin][idx.local - s]); - } - else - { - buffer[iout][idx.local] = buffer[iin][idx.local]; - } - __syncthreads(); - } - x = buffer[iout][idx.nlocal() - 1]; - output(i, buffer[iout][idx.local]); - }); -} - -template -__device__ void block_scan(index idx, Op op, T init, index_int n, Input input, Output output) -{ - block_scan( - idx, - op, - init, - [&](auto f) -> decltype(f(index_int{})) { return idx.local_stride(n, f); }, - input, - output); -} - -template -constexpr auto reverse_scan(index_int n, F f) -{ - return [=](auto i, auto&&... xs) { return f(n - i - 1, xs...); }; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_DEVICE_SCAN_HPP diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/shape.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/shape.hpp deleted file mode 100644 index 66f065c78..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/shape.hpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_SHAPE_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_SHAPE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -struct hip_shape -{ - using hip_index = hip_array; - hip_index lens = {}; - hip_index strides = {}; - hip_array divs = {}; - bool standard = false; - - __device__ __host__ hip_shape() = default; - - hip_shape(const shape& s) : standard(s.standard()) - { - assert(s.lens().size() == N); - assert(s.strides().size() == N); - std::copy(s.lens().begin(), s.lens().end(), lens.begin()); - std::copy(s.strides().begin(), s.strides().end(), strides.begin()); - assert(std::all_of(s.lens().begin(), s.lens().end(), &is_divisor_encodable)); - std::transform(s.lens().begin(), s.lens().end(), divs.begin(), &encode_divisor); - } - - MIGRAPHX_DEVICE_CONSTEXPR index_int elements() const { return lens.product(); } - - MIGRAPHX_DEVICE_CONSTEXPR index_int index(hip_index x) const { return x.dot(strides); } - - MIGRAPHX_DEVICE_CONSTEXPR index_int index(std::initializer_list x) const - { - index_int idx = 0; - for(index_int i = 0; i < x.size(); i++) - idx += *(x.begin() + i) * strides[i]; - return idx; - } - - MIGRAPHX_DEVICE_CONSTEXPR index_int index(index_int i) const - { - if(this->standard) - return i; - else - { - const index_int rank = this->lens.size(); - index_int s = 1; - index_int result = 0; - for(index_int j = 0; j < this->lens.size(); j++) - { - const index_int k = rank - j - 1; - const index_int stride = this->strides[k]; - const index_int len = this->lens[k]; - const index_int slen = s * len; - const index_int idx = (i % slen) / s; - result += stride * idx; - s = slen; - } - return result; - } - } - - MIGRAPHX_DEVICE_CONSTEXPR hip_index multi(index_int idx) const - { - hip_index result; - index_int tidx = idx; - for(std::ptrdiff_t is = result.size() - 1; is > 0; is--) - { - // result[is] = tidx % lens[is]; - // tidx = tidx / lens[is]; - auto q = fast_div(tidx, divs[is]); - result[is] = remainder(q, tidx, lens[is]); - tidx = q; - } - result[0] = tidx; - return result; - } -}; - -template -hip_shape make_hip_shape(const shape& x) -{ - return x; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/tensor.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/tensor.hpp deleted file mode 100644 index 2b85cb89d..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/tensor.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEAVICE_TENSOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEAVICE_TENSOR_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -using hip_tensor_index = hip_array; - -template -struct hip_tensor_descriptor -{ - __device__ __host__ hip_tensor_descriptor() = default; - - hip_tensor_descriptor(const shape& s) - { - std::copy(s.lens().begin(), s.lens().end(), lens); - std::copy(s.strides().begin(), s.strides().end(), strides); - } - - __device__ __host__ hip_tensor_index multi(index_int idx) const - { - hip_tensor_index result{}; - index_int tidx = idx; - for(index_int is = 0; is < NDim; is++) - { - result[is] = tidx / strides[is]; - tidx = tidx % strides[is]; - } - - return result; - } - __device__ __host__ index_int linear(hip_tensor_index s) const - { - index_int idx = 0; - for(index_int i = 0; i < NDim; i++) - idx += s[i] * strides[i]; - return idx; - } - index_int lens[NDim] = {}; - index_int strides[NDim] = {}; -}; - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/tensor_view.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/tensor_view.hpp deleted file mode 100644 index 8be3908a4..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/tensor_view.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_TENSOR_VIEW_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_TENSOR_VIEW_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -struct hip_tensor_view -{ - using value_type = T; - using hip_index = typename hip_shape::hip_index; - __device__ __host__ hip_tensor_view() = default; - __host__ hip_tensor_view(tensor_view x) : d(x.data()), s(x.get_shape()) {} - __host__ hip_tensor_view(T* x, const shape& ss) : d(x), s(ss) {} - - MIGRAPHX_DEVICE_CONSTEXPR const hip_shape& get_shape() const { return s; } - - MIGRAPHX_DEVICE_CONSTEXPR index_int size() const { return s.elements(); } - - MIGRAPHX_DEVICE_CONSTEXPR value_type* data() const { return d; } - - template - MIGRAPHX_DEVICE_CONSTEXPR value_type& operator[](U i) const - { - return d[s.index(i)]; - } - - MIGRAPHX_DEVICE_CONSTEXPR value_type* begin() const { return d; } - - MIGRAPHX_DEVICE_CONSTEXPR value_type* end() const { return d + size(); } - - private: - value_type* d = nullptr; - hip_shape s{}; -}; - -template -hip_tensor_view make_hip_view(const shape& s, T* x) -{ - return {x, s}; -} - -template -hip_tensor_view make_hip_view(tensor_view x) -{ - return {x}; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/types.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/types.hpp deleted file mode 100644 index c9f2e3d7c..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/types.hpp +++ /dev/null @@ -1,213 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_DEVICE_TYPES_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_DEVICE_TYPES_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -using index_int = std::uint32_t; - -#define MIGRAPHX_DEVICE_CONSTEXPR constexpr __device__ __host__ // NOLINT - -template -using vec = T __attribute__((ext_vector_type(N))); - -template -__device__ __host__ T* as_pointer(vec* x) -{ - return reinterpret_cast(x); -} - -template -__device__ __host__ vec* as_vec(T* x) -{ - return reinterpret_cast*>(x); -} - -template -tensor_view> as_vec(tensor_view x) -{ - return {x.get_shape(), as_vec(x.data())}; -} - -template -auto pack_vec(Ts... xs) -{ - return [=](auto f, index_int n) { return f(as_vec(xs)[n]...); }; -} - -using gpu_half = __fp16; -using gpu_bf16 = __bf16; - -namespace detail { -template -struct device_type -{ - using type = T; -}; - -template -struct device_type> -{ - using type = vec::type, N>; -}; - -template <> -struct device_type -{ - using type = gpu_half; -}; - -template <> -struct device_type -{ - using type = gpu_bf16; -}; - -template -struct host_type -{ - using type = T; -}; - -template <> -struct host_type -{ - using type = half; -}; - -template <> -struct host_type -{ - using type = bf16; -}; - -} // namespace detail - -template -using host_type = typename detail::host_type::type; - -template -using device_type = typename detail::device_type::type; - -template -host_type host_cast(T x) -{ - return reinterpret_cast&>(x); -} - -template -host_type* host_cast(T* x) -{ - return reinterpret_cast*>(x); -} - -template -__device__ __host__ device_type device_cast(const T& x) -{ - return reinterpret_cast&>(x); -} - -template -__device__ __host__ device_type* device_cast(T* x) -{ - return reinterpret_cast*>(x); -} - -template -__device__ __host__ tensor_view> device_cast(tensor_view x) -{ - return {x.get_shape(), reinterpret_cast*>(x.data())}; -} - -template -__device__ __host__ T to_hip_type(T x) -{ - return x; -} - -// Hip doens't support __fp16 and __bf16 -inline __device__ __host__ float to_hip_type(gpu_half x) { return x; } -inline __device__ __host__ float to_hip_type(gpu_bf16 x) { return x; } - -template -struct is_floating_point : std::is_floating_point -{ -}; - -template <> -struct is_floating_point<__fp16> : std::true_type -{ -}; - -template -struct is_signed : std::is_signed -{ -}; - -template <> -struct is_signed<__fp16> : std::true_type -{ -}; - -template -struct is_arithmetic : std::is_arithmetic -{ -}; - -template <> -struct is_arithmetic<__fp16> : std::true_type -{ -}; - -// Redo for __bf16 -template <> -struct is_floating_point<__bf16> : std::true_type -{ -}; -template <> -struct is_signed<__bf16> : std::true_type -{ -}; -template <> -struct is_arithmetic<__bf16> : std::true_type -{ -}; - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/vector.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/vector.hpp deleted file mode 100644 index 93fe06b0c..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/vector.hpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_VECTOR_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_VECTOR_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -struct hip_vector -{ - MIGRAPHX_DEVICE_CONSTEXPR hip_vector() = default; - MIGRAPHX_DEVICE_CONSTEXPR hip_vector(index_int s) : len(s) {} - template - __device__ __host__ hip_vector(Iterator start, Iterator last) - { - auto it = std::copy(start, last, d); - len = std::distance(d, it); - } - - __device__ __host__ hip_vector(std::initializer_list x) - { - std::copy(x.begin(), x.end(), d); - len = x.size(); - } - - MIGRAPHX_DEVICE_CONSTEXPR T& operator[](index_int i) { return d[i]; } - MIGRAPHX_DEVICE_CONSTEXPR const T& operator[](index_int i) const { return d[i]; } - - MIGRAPHX_DEVICE_CONSTEXPR T& front() { return d[0]; } - MIGRAPHX_DEVICE_CONSTEXPR const T& front() const { return d[0]; } - - MIGRAPHX_DEVICE_CONSTEXPR T& back() { return d[size() - 1]; } - MIGRAPHX_DEVICE_CONSTEXPR const T& back() const { return d[size() - 1]; } - - MIGRAPHX_DEVICE_CONSTEXPR T* data() { return d; } - MIGRAPHX_DEVICE_CONSTEXPR const T* data() const { return d; } - - MIGRAPHX_DEVICE_CONSTEXPR index_int size() const { return len; } - - MIGRAPHX_DEVICE_CONSTEXPR T* begin() { return d; } - MIGRAPHX_DEVICE_CONSTEXPR const T* begin() const { return d; } - - MIGRAPHX_DEVICE_CONSTEXPR T* end() { return d + size(); } - MIGRAPHX_DEVICE_CONSTEXPR const T* end() const { return d + size(); } - - template - MIGRAPHX_DEVICE_CONSTEXPR void push_back(U&& x) - { - d[len] = static_cast(x); - len++; - } - - private: - T d[N] = {}; - index_int len = 0; -}; - -template -hip_vector to_hip_vector(const std::vector& x) -{ - hip_vector result(x.size()); - std::copy(x.begin(), x.end(), result.begin()); - return result; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/visit.hpp b/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/visit.hpp deleted file mode 100644 index 78f28a552..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/include/migraphx/gpu/device/visit.hpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_VISIT_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_VISIT_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -constexpr void visit_tensor_size(index_int n, F f) -{ - switch(n) - { - case 1: { - f(std::integral_constant{}); - break; - } - case 2: { - f(std::integral_constant{}); - break; - } - case 3: { - f(std::integral_constant{}); - break; - } - case 4: { - f(std::integral_constant{}); - break; - } - case 5: { - f(std::integral_constant{}); - break; - } - default: throw std::runtime_error("Tensor dims " + std::to_string(n) + " out of range"); - } -} - -inline shape get_shape(const shape& x) { return x; } - -template -auto get_shape(const T& x) -> decltype(x.get_shape()) -{ - return x.get_shape(); -} - -template -struct is_hip_type : std::false_type -{ -}; - -template <> -struct is_hip_type : std::true_type -{ -}; -template <> -struct is_hip_type : std::true_type -{ -}; -template <> -struct is_hip_type : std::true_type -{ -}; -template <> -struct is_hip_type : std::true_type -{ -}; -template <> -struct is_hip_type : std::true_type -{ -}; -template <> -struct is_hip_type : std::true_type -{ -}; -template <> -struct is_hip_type : std::true_type -{ -}; - -template {})> -void hip_visitor_invoke(T as, V&& v) -{ - v(as); -} - -template {})> -void hip_visitor_invoke(T, V&&) -{ - MIGRAPHX_THROW(std::string("Unsupported data type on GPU: ") + __PRETTY_FUNCTION__); -} - -template -auto hip_visitor(V v) -{ - return [=](auto as) { hip_visitor_invoke(as, v); }; -} - -template -void hip_visit_all_impl(const shape& s, F f, V&& v, Ts&&... xs) -{ - std::initializer_list types = {get_shape(xs).type()...}; - if(not std::all_of( - types.begin(), types.end(), [&](migraphx::shape::type_t t) { return t == s.type(); })) - MIGRAPHX_THROW("Types must be the same"); - std::initializer_list ranks = {static_cast(get_shape(xs).ndim())...}; - if(not std::all_of(ranks.begin(), ranks.end(), [&](index_int r) { return r == s.ndim(); })) - MIGRAPHX_THROW("Ranks must be the same"); - visit_tensor_size(s.ndim(), [&](auto ndim) { - s.visit_type(hip_visitor([&](auto as) { v(f(xs, ndim, as)...); })); - }); -} - -template -void hip_visit_views_impl(const shape& s, F f, V&& v, Ts&&... xs) -{ - std::initializer_list ranks = {static_cast(get_shape(xs).ndim())...}; - if(not std::all_of(ranks.begin(), ranks.end(), [&](index_int r) { return r == s.ndim(); })) - MIGRAPHX_THROW("Ranks must be the same"); - visit_tensor_size(s.ndim(), [&](auto ndim) { v(f(xs, ndim)...); }); -} - -template -struct hip_convert -{ - F f; - template - auto operator()(RawData x, N ndim, As as) const - -> decltype(make_hip_view(x.get_shape(), f(as.from(x.data())))) - { - return make_hip_view(x.get_shape(), f(as.from(x.data()))); - } - - template - auto operator()(const shape& s, N ndim, As) const - { - return make_hip_shape(s); - } -}; - -template -hip_convert make_hip_convert(F f) -{ - return {f}; -} - -template -struct hip_convert_view -{ - F f; - template - auto operator()(tensor_view x, N ndim) const - { - return make_hip_view(f(x)); - } - - template - auto operator()(const shape& s, N ndim) const - { - return make_hip_shape(s); - } -}; - -template -hip_convert_view make_hip_convert_view(F f) -{ - return {f}; -} - -template -auto hip_visit_all(T&& x, Ts&&... xs) -{ - return [&](auto f) { - hip_visit_all_impl( - get_shape(x), make_hip_convert([](auto* p) { return device_cast(p); }), f, x, xs...); - }; -} - -template -auto hip_vec_visit_all(T&& x, Ts&&... xs) -{ - return [&](auto f) { - auto sx = get_shape(x); - auto lens = sx.lens(); - assert(lens.back() % N == 0); - assert(sx.strides().back() == 1); - lens.back() /= N; - shape vec_sx{sx.type(), lens}; - hip_visit_all_impl(vec_sx, - make_hip_convert([](auto* p) { return as_vec(device_cast(p)); }), - f, - x, - xs...); - }; -} - -template -auto hip_pointer_visit_all(T&& x, Ts&&... xs) -{ - return [&](auto f) { visit_all(x, xs...)([&](auto... vs) { f(device_cast(vs.data())...); }); }; -} - -template -auto hip_visit_views(T&& x, Ts&&... xs) -{ - return [&](auto f) { - hip_visit_views_impl(get_shape(x), - make_hip_convert_view([](auto v) { return device_cast(v); }), - f, - x, - xs...); - }; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/device/logsoftmax.cpp b/docker/rocm/migraphx/targets/gpu/device/logsoftmax.cpp deleted file mode 100644 index f2dd6148b..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/logsoftmax.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void logsoftmax(hipStream_t stream, const argument& result, const argument& arg, int64_t axis) -{ - auto batch_lens = result.get_shape().lens(); - index_int batch_item_num = batch_lens[axis]; - batch_lens[axis] = 1; - migraphx::shape batch_shape{result.get_shape().type(), batch_lens}; - - hip_visit_all(result, arg, batch_shape)([&](auto output, auto input, auto batch) { - const index_int max_block_size = 256; - const index_int block_size = compute_block_size(batch_item_num, max_block_size); - gs_launch(stream, - batch_shape.elements() * block_size, - block_size)([=](auto i, auto idx) __device__ { - auto data_idx = batch.multi(i / block_size); - using type = device_type>; - type init = lowest(); - - auto batch_max = block_reduce( - idx, max{}, init, batch_item_num, [&](auto j) __device__ { - data_idx[axis] = j; - return input[data_idx]; - }); - - auto batch_sum = - block_reduce(idx, sum{}, 0, batch_item_num, [&](auto j) __device__ { - data_idx[axis] = j; - auto val = input[data_idx] - batch_max; - return ::exp(to_hip_type(val)); - }); - - auto log_batch_sum = ::log(to_hip_type(batch_sum)) + batch_max; - - idx.local_stride(batch_item_num, [&](auto j) __device__ { - data_idx[axis] = j; - output[data_idx] = input[data_idx] - log_batch_sum; - }); - }); - }); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/multinomial.cpp b/docker/rocm/migraphx/targets/gpu/device/multinomial.cpp deleted file mode 100644 index e7a89d7f1..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/multinomial.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -constexpr Iterator upper_bound(Iterator first, Iterator last, const T& value) -{ - Iterator it; - typename std::iterator_traits::difference_type count; - typename std::iterator_traits::difference_type step; - count = std::distance(first, last); - - while(count > 0) - { - it = first; - step = count / 2; - std::advance(it, step); - if(not(value < *it)) - { - first = ++it; - count -= step + 1; - } - else - count = step; - } - return first; -} - -void multinomial(hipStream_t stream, - const argument& result, - const argument& arg0, - const argument& arg1) -{ - size_t batch_size = arg0.get_shape().lens().front(); - size_t class_size = arg0.get_shape().lens().back(); - size_t sample_size = result.get_shape().lens().back(); - - visit_all(arg0, arg1)([&](auto cdf_host, auto dist_host) { - result.visit([&](auto output_host) { - hip_visit_views(cdf_host, dist_host, output_host)( - [&](auto cdf, auto dist, auto output) { - gs_launch(stream, batch_size * sample_size)([=](auto i) __device__ { - auto idx = output.get_shape().multi(i); - auto cdf_begin = cdf.begin() + (idx.front() * class_size); - auto cdf_end = cdf_begin + class_size; - auto* sample_iter = - upper_bound(cdf_begin, cdf_end, dist[i] * *(std::prev(cdf_end))); - output[i] = std::distance(cdf_begin, sample_iter); - }); - }); - }); - }); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/nonzero.cpp b/docker/rocm/migraphx/targets/gpu/device/nonzero.cpp deleted file mode 100644 index 223713390..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/nonzero.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -argument nonzero(hipStream_t stream, const argument& result, const argument& arg_data) -{ - auto s = arg_data.get_shape(); - auto elem_num = s.elements(); - auto out_elem_num = result.get_shape().elements(); - - // call the prefix_sum function to do a prefix_sum to compute - // index in the output. Only 1 block can be used since we have - // only one prefix sum - const index_int block_size = 256; - hip_visit_all(arg_data, s)([&](auto input, auto si) { - const auto* in_ptr = device_cast(input.data()); - auto* ptr = result.cast(); - gs_launch(stream, block_size, block_size)([=](auto, auto idx) __device__ { - // fill all output to 0 first - idx.local_stride(out_elem_num, [&](auto j) { ptr[j] = 0; }); - - block_scan( - idx, - sum{}, - 0, - elem_num, - [&](auto j) { return (float_equal(in_ptr[j], 0)) ? 0 : 1; }, - [&](auto j, auto x) { - auto out_loc = x - 1; - if(float_equal(in_ptr[j], 0)) - return; - - auto index = si.multi(j); - for(size_t k = 0; k < index.size(); ++k) - { - ptr[k * elem_num + out_loc] = index[k]; - } - }); - }); - }); - - return result; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/prefix_scan_sum.cpp b/docker/rocm/migraphx/targets/gpu/device/prefix_scan_sum.cpp deleted file mode 100644 index 9518f5b45..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/prefix_scan_sum.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void prefix_scan_sum(hipStream_t stream, - const argument& result, - const argument& arg, - int32_t axis, - bool exclusive, - bool reverse) -{ - const index_int max_block_size = 256; - const index_int n = arg.get_shape().lens()[axis]; - auto rlens = result.get_shape().lens(); - rlens[axis] = 1; - - hip_visit_all(result, arg, result.get_shape().with_lens(rlens))( - [=](auto output, auto input, auto rshape) { - const index_int block_size = compute_block_size(rshape.elements(), max_block_size); - if(reverse and exclusive) - { - gs_launch(stream, rshape.elements() * block_size, block_size)( - [=](auto i, auto idx) __device__ { - const auto ridx = rshape.multi(i / block_size); - auto compute_idx = [&](auto j) { - auto k = ridx; - k[axis] = j; - return k; - }; - block_scan( - idx, - sum{}, - 0, - n, - reverse_scan(n, [&](auto j) { return input[compute_idx(j)]; }), - reverse_scan(n, [&](auto j, auto x) { - if(j == n - 1) - output[compute_idx(j)] = 0; - if(j > 0) - output[compute_idx(j - 1)] = x; - })); - }); - } - else if(reverse) - { - gs_launch(stream, rshape.elements() * block_size, block_size)( - [=](auto i, auto idx) __device__ { - const auto ridx = rshape.multi(i / block_size); - auto compute_idx = [&](auto j) { - auto k = ridx; - k[axis] = j; - return k; - }; - block_scan( - idx, - sum{}, - 0, - n, - reverse_scan(n, [&](auto j) { return input[compute_idx(j)]; }), - reverse_scan(n, [&](auto j, auto x) { output[compute_idx(j)] = x; })); - }); - } - else if(exclusive) - { - gs_launch(stream, rshape.elements() * block_size, block_size)( - [=](auto i, auto idx) __device__ { - const auto ridx = rshape.multi(i / block_size); - auto compute_idx = [&](auto j) { - auto k = ridx; - k[axis] = j; - return k; - }; - block_scan( - idx, - sum{}, - 0, - n, - [&](auto j) { return input[compute_idx(j)]; }, - [&](auto j, auto x) { - auto k = j + 1; - if(j == 0) - output[compute_idx(0)] = 0; - if(k < n) - output[compute_idx(k)] = x; - }); - }); - } - else - { - gs_launch(stream, rshape.elements() * block_size, block_size)( - [=](auto i, auto idx) __device__ { - const auto ridx = rshape.multi(i / block_size); - auto compute_idx = [&](auto j) { - auto k = ridx; - k[axis] = j; - return k; - }; - block_scan( - idx, - sum{}, - 0, - n, - [&](auto j) { return input[compute_idx(j)]; }, - [&](auto j, auto x) { output[compute_idx(j)] = x; }); - }); - } - }); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/reverse.cpp b/docker/rocm/migraphx/targets/gpu/device/reverse.cpp deleted file mode 100644 index 5d5831127..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/reverse.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "migraphx/gpu/device/visit.hpp" -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -argument -reverse(hipStream_t stream, argument result, argument arg1, const std::vector& axes) -{ - auto s = arg1.get_shape(); - // auto lens = s.lens(); - std::vector axis_len(axes.begin(), axes.end()); - shape sa{shape::float_type, axis_len}; - std::size_t nelements = s.elements(); - visit_all(result, arg1)([&](auto output1, auto input1) { - hip_visit_views(output1, input1, s)([&](auto output, auto input, auto hs) { - hip_visit_views(sa)([&](auto daxes) { - auto lens = hs.lens; - gs_launch(stream, nelements)([=](auto i) __device__ { - auto idx = hs.multi(i); - auto in_idx = idx; - for(auto axis : daxes.lens) - in_idx[axis] = lens[axis] - 1 - idx[axis]; - output[idx] = input[in_idx]; - }); - }); - }); - }); - - return result; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/rnn_variable_seq_lens.cpp b/docker/rocm/migraphx/targets/gpu/device/rnn_variable_seq_lens.cpp deleted file mode 100644 index 6d21c702f..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/rnn_variable_seq_lens.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void rnn_var_sl_shift_sequence(hipStream_t stream, - const argument& result, - const argument& arg_hs, - const argument& arg_sl) -{ - auto output_shape = result.get_shape(); - int64_t max_len = output_shape.lens()[0]; - visit_all(result, arg_hs)([&](auto output, auto input) { - const auto* in_data = device_cast(input.data()); - auto* out_data = device_cast(output.data()); - auto out_s = make_hip_shape<3>(output_shape); - arg_sl.visit([&](auto sl) { - const auto* sl_data = device_cast(sl.data()); - gs_launch(stream, output_shape.elements(), 256)([=](auto i) __device__ { - auto idx = out_s.multi(i); - auto t = idx[0]; - auto b = idx[1]; - auto l = sl_data[b]; - auto val = in_data[0]; - val = 0; - if(t >= max_len - l) - { - auto in_idx = idx; - in_idx[0] -= (max_len - l); - val = in_data[out_s.index(in_idx)]; - } - out_data[i] = val; - }); - }); - }); -} - -void rnn_var_sl_shift_output(hipStream_t stream, - const argument& result, - const argument& arg_hs, - const argument& arg_sl, - bool is_reverse) -{ - auto output_shape = result.get_shape(); - int64_t max_len = output_shape.lens()[0]; - visit_all(result, arg_hs)([&](auto output, auto input) { - const auto* in_data = device_cast(input.data()); - auto* out_data = device_cast(output.data()); - auto out_s = make_hip_shape<4>(output_shape); - arg_sl.visit([&](auto sl) { - const auto* sl_data = device_cast(sl.data()); - gs_launch(stream, output_shape.elements(), 256)([=](auto i) __device__ { - auto idx = out_s.multi(i); - auto t = idx[0]; - auto d = idx[1]; - auto b = idx[2]; - auto l = sl_data[b]; - auto val = in_data[0]; - val = 0; - if(t < l) - { - int offset = (d == 1 or is_reverse) ? 1 : 0; - auto in_idx = idx; - in_idx[0] += offset * (max_len - l); - val = in_data[out_s.index(in_idx)]; - } - out_data[i] = val; - }); - }); - }); -} - -void rnn_var_sl_last_output(hipStream_t stream, - const argument& result, - const argument& arg_hs, - const argument& arg_sl, - bool is_reverse) -{ - auto input_shape = arg_hs.get_shape(); - auto out_comp_lens = input_shape.lens(); - out_comp_lens[0] = 1; - shape out_comp_shape{input_shape.type(), out_comp_lens}; - - visit_all(result, arg_hs)([&](auto output, auto input) { - const auto* in_data = device_cast(input.data()); - auto* out_data = device_cast(output.data()); - arg_sl.visit([&](auto sl) { - const auto* sl_data = device_cast(sl.data()); - auto in_s = make_hip_shape<4>(input_shape); - auto out_s = make_hip_shape<4>(out_comp_shape); - gs_launch(stream, result.get_shape().elements(), 256)([=](auto i) __device__ { - auto idx = out_s.multi(i); - auto d = idx[1]; - auto b = idx[2]; - auto l = sl_data[b]; - if(is_reverse or d == 1) - { - idx[0] = 0; - } - else - { - idx[0] = l - 1; - } - out_data[i] = in_data[in_s.index(idx)]; - }); - }); - }); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/targets.cpp b/docker/rocm/migraphx/targets/gpu/device/targets.cpp deleted file mode 100644 index 0b1853db7..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/targets.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -static std::vector parse_targets() { return split_string(MIGRAPHX_GPU_TARGETS, ';'); } - -const std::vector& get_targets() -{ - static auto result = parse_targets(); - return result; -} - -std::string get_targets_as_string() { return join_strings(get_targets(), ", "); } - -static int get_device_id() -{ - int device; - auto status = hipGetDevice(&device); - if(status != hipSuccess) - MIGRAPHX_THROW("No device"); - return device; -} - -std::string get_device_name() -{ - hipDeviceProp_t props{}; - auto status = hipGetDeviceProperties(&props, get_device_id()); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to get device properties"); - return props.gcnArchName; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device/targets.hpp.in b/docker/rocm/migraphx/targets/gpu/device/targets.hpp.in deleted file mode 100644 index 0a0e19aba..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/targets.hpp.in +++ /dev/null @@ -1,52 +0,0 @@ -/* -* The MIT License (MIT) -* -* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in -* all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -* THE SOFTWARE. -*/ -#ifndef MIGRAPHX_GUARD_DEVICE_TARGETS_CPP -#define MIGRAPHX_GUARD_DEVICE_TARGETS_CPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { -#define MIGRAPHX_GPU_TARGETS "@GPU_TARGETS@" // NOLINT - -MIGRAPHX_DEVICE_EXPORT -const std::vector& get_targets(); - -MIGRAPHX_DEVICE_EXPORT -std::string get_targets_as_string(); - -MIGRAPHX_DEVICE_EXPORT -std::string get_device_name(); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_DEVICE_TARGETS_CPP - - diff --git a/docker/rocm/migraphx/targets/gpu/device/topk.cpp b/docker/rocm/migraphx/targets/gpu/device/topk.cpp deleted file mode 100644 index 2168af94e..000000000 --- a/docker/rocm/migraphx/targets/gpu/device/topk.cpp +++ /dev/null @@ -1,239 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -struct hip_heap_vector -{ - MIGRAPHX_DEVICE_CONSTEXPR hip_heap_vector(T* val, index_int n, Index v_idx, Compare comp) - : data(val), size(n), data_index(v_idx), compare(comp) - { - make_heap(size); - } - - MIGRAPHX_DEVICE_CONSTEXPR void try_push(const T val) - { - if(compare(val, data[data_index(0)])) - return; - - pop_heap(size - 1); - data[data_index(size - 1)] = val; - push_heap(size - 1); - } - - MIGRAPHX_DEVICE_CONSTEXPR void sort() { sort_heap(size); } - - private: - MIGRAPHX_DEVICE_CONSTEXPR inline static void swap(T& v1, T& v2) noexcept - { - T v = v1; - v1 = v2; - v2 = v; - } - - MIGRAPHX_DEVICE_CONSTEXPR inline void heapify_down(index_int n, index_int index) - { - while(index < n) - { - auto pre_index = index; - index_int l = 2 * index + 1; - index_int r = 2 * index + 2; - - if(l < n and compare(data[data_index(l)], data[data_index(index)])) - { - index = l; - } - - if(r < n and compare(data[data_index(r)], data[data_index(index)])) - { - index = r; - if(compare(data[data_index(l)], data[data_index(r)])) - { - index = l; - } - } - - if(index == pre_index) - { - break; - } - - swap(data[data_index(index)], data[data_index(pre_index)]); - } - } - - MIGRAPHX_DEVICE_CONSTEXPR inline void heapify_up(index_int index) - { - while(index > 0) - { - auto parent_idx = (index - 1) / 2; - - if(not compare(data[data_index(index)], data[data_index(parent_idx)])) - { - break; - } - - swap(data[data_index(index)], data[data_index(parent_idx)]); - index = parent_idx; - } - } - - MIGRAPHX_DEVICE_CONSTEXPR inline void make_heap(index_int n) - { - for(int j = n / 2 - 1; j >= 0; --j) - { - heapify_down(n, j); - } - } - - MIGRAPHX_DEVICE_CONSTEXPR inline void push_heap(index_int loc) { heapify_up(loc); } - - MIGRAPHX_DEVICE_CONSTEXPR inline void pop_heap(index_int loc) - { - swap(data[data_index(0)], data[data_index(loc)]); - heapify_down(loc, 0); - } - - MIGRAPHX_DEVICE_CONSTEXPR inline void sort_heap(index_int n) - { - for(int j = n - 1; j > 0; --j) - { - swap(data[data_index(0)], data[data_index(j)]); - heapify_down(j, 0); - } - } - - T* data = nullptr; - index_int size; - Index data_index; - Compare compare; -}; - -template -__device__ hip_heap_vector -make_heap(T* data, index_int n, Index idx, Compare compare) -{ - return {data, n, idx, compare}; -} - -template -std::vector topk(hipStream_t stream, - const argument& val_res, - const argument& ind_res, - const argument& arg, - int64_t k, - int64_t axis, - Compare compare) -{ - auto in_s = arg.get_shape(); - auto in_lens = in_s.lens(); - auto out_s = val_res.get_shape(); - auto axis_dim = in_s.lens()[axis]; - auto comp_lens = in_lens; - comp_lens[axis] = 1; - shape comp_s{in_s.type(), comp_lens}; - std::size_t elem_num = comp_s.elements(); - - hip_visit_all(val_res, arg, out_s, in_s, comp_s)( - [&](auto out_val, auto input, auto oss, auto iss, auto css) { - auto* data = device_cast(input.data()); - auto* out = device_cast(out_val.data()); - auto* const ind = ind_res.cast(); - gs_launch(stream, elem_num)([=](auto i) __device__ { - auto idx = css.multi(i); - - auto in_idx = [&](int ii) { - auto iidx = idx; - iidx[axis] = ii; - return iss.index(iidx); - }; - - auto out_idx = [&](int ii) { - auto iidx = idx; - iidx[axis] = ii; - return oss.index(iidx); - }; - - auto data_compare = [=](auto ii, auto jj) { - return compare(data[in_idx(ii)], data[in_idx(jj)]); - }; - - for(int j = 0; j < k; ++j) - { - ind[out_idx(j)] = j; - } - - auto hp = make_heap(ind, k, out_idx, data_compare); - for(int j = k; j < axis_dim; ++j) - { - hp.try_push(j); - } - hp.sort(); - - for(int j = 0; j < k; ++j) - { - out[out_idx(j)] = data[in_idx(ind[out_idx(j)])]; - } - }); - }); - - return {val_res, ind_res}; -} - -argument topk_largest(hipStream_t stream, - const argument& val_res, - const argument& ind_res, - const argument& arg, - int64_t k, - int64_t axis) -{ - return {topk(stream, val_res, ind_res, arg, k, axis, std::less<>{})}; -} - -argument topk_smallest(hipStream_t stream, - const argument& val_res, - const argument& ind_res, - const argument& arg, - int64_t k, - int64_t axis) -{ - return {topk(stream, val_res, ind_res, arg, k, axis, std::greater<>{})}; -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/device_name.cpp b/docker/rocm/migraphx/targets/gpu/device_name.cpp deleted file mode 100644 index c717742e2..000000000 --- a/docker/rocm/migraphx/targets/gpu/device_name.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -int get_device_id() -{ - int device; - auto status = hipGetDevice(&device); - if(status != hipSuccess) - MIGRAPHX_THROW("No device"); - return device; -} - -std::string get_device_name() -{ - hipDeviceProp_t props{}; - auto status = hipGetDeviceProperties(&props, get_device_id()); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to get device properties"); - return props.gcnArchName; -} - -bool gfx_has_fp8fnuz_intrinsics() -{ - const auto device_name = trim(split_string(get_device_name(), ':').front()); - return (starts_with(device_name, "gfx94")); -} - -bool gfx_has_fp8ocp_intrinsics() -{ - const auto device_name = trim(split_string(get_device_name(), ':').front()); - bool is_navi_with_fp8ocp = starts_with(device_name, "gfx12") and device_name >= "gfx1200"; - bool is_mi_with_fp8ocp = starts_with(device_name, "gfx9") and device_name >= "gfx950"; - return (is_navi_with_fp8ocp or is_mi_with_fp8ocp); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/driver/CMakeLists.txt b/docker/rocm/migraphx/targets/gpu/driver/CMakeLists.txt deleted file mode 100644 index ae9b9a685..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -file(GLOB GPU_DRIVER_SRCS CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp) -add_executable(gpu-driver - ${GPU_DRIVER_SRCS} -) -rocm_clang_tidy_check(gpu-driver) -target_include_directories(gpu-driver PRIVATE include) -target_link_libraries(gpu-driver PRIVATE migraphx_gpu) diff --git a/docker/rocm/migraphx/targets/gpu/driver/action.cpp b/docker/rocm/migraphx/targets/gpu/driver/action.cpp deleted file mode 100644 index ea71afdf1..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/action.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -auto& action_map() -{ - static std::unordered_map m; - return m; -} - -action_function get_action(const std::string& name) -{ - if(action_map().count(name) == 0) - MIGRAPHX_THROW("Missing action: " + name); - return action_map().at(name); -} - -void register_action(const std::string& name, const action_function& a) { action_map()[name] = a; } - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/driver/compile_op.cpp b/docker/rocm/migraphx/targets/gpu/driver/compile_op.cpp deleted file mode 100644 index 5caae2a79..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/compile_op.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -struct compile_op : action -{ - static void apply(const parser& p, const value& v) - { - context ctx; - auto inputs = p.parse_shapes(v.at("inputs")); - auto op = gpu::compile_op(v.at("name").to(), ctx, inputs, v); - auto t = time_op(ctx, op, inputs, p.get(v, "iterations", 100)); - std::cout << op << " -> " << op.compute_shape(inputs) << ": " << t << "ms" << std::endl; - std::cout << std::endl; - } -}; - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/driver/include/migraphx/gpu/driver/action.hpp b/docker/rocm/migraphx/targets/gpu/driver/include/migraphx/gpu/driver/action.hpp deleted file mode 100644 index 172419e7c..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/include/migraphx/gpu/driver/action.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_DRIVER_ACTION_HPP -#define MIGRAPHX_GUARD_GPU_DRIVER_ACTION_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -using action_function = std::function; - -action_function get_action(const std::string& name); -void register_action(const std::string& name, const action_function& a); - -struct auto_register_action -{ - template - static void apply() - { - const auto& name = get_type_name(); - register_action(name.substr(name.rfind("::") + 2), - [](auto&&... xs) { T::apply(std::forward(xs)...); }); - } -}; - -template -using action = auto_register; - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_DRIVER_ACTION_HPP diff --git a/docker/rocm/migraphx/targets/gpu/driver/include/migraphx/gpu/driver/parser.hpp b/docker/rocm/migraphx/targets/gpu/driver/include/migraphx/gpu/driver/parser.hpp deleted file mode 100644 index d5995eeb5..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/include/migraphx/gpu/driver/parser.hpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_DRIVER_PARSER_HPP -#define MIGRAPHX_GUARD_GPU_DRIVER_PARSER_HPP - -#include -#include - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -[[noreturn]] void error(const std::string& msg); - -struct parser -{ - parser() = default; - - template - T get(const value& v, const std::string& key, const T& default_value) const - { - return v.get(key, settings.get(key, default_value)); - } - - shape parse_shape(const value& v) const; - - std::vector parse_shapes(const value& v) const; - - void load_settings(const value& v); - - static void process(const value& v); - - private: - value settings = value::object{}; -}; - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_DRIVER_PARSER_HPP diff --git a/docker/rocm/migraphx/targets/gpu/driver/main.cpp b/docker/rocm/migraphx/targets/gpu/driver/main.cpp deleted file mode 100644 index c61e447db..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/main.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -using namespace migraphx; // NOLINT -using namespace migraphx::gpu; // NOLINT -using namespace migraphx::gpu::driver; // NOLINT - -int main(int argc, char const* argv[]) -{ - std::vector args(argv, argv + argc); - if(args.size() < 2) - { - std::cout << "Usage: gpu-driver " << std::endl; - std::abort(); - } - auto v = from_json_string(convert_to_json(read_string(args[1]))); - parser::process(v); -} diff --git a/docker/rocm/migraphx/targets/gpu/driver/parser.cpp b/docker/rocm/migraphx/targets/gpu/driver/parser.cpp deleted file mode 100644 index c84d00580..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/parser.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -[[noreturn]] void error(const std::string& msg) -{ - std::cout << msg << std::endl; - std::abort(); -} - -shape parser::parse_shape(const value& v) const -{ - auto lens = get(v, "lens", std::vector{}); - auto strides = get(v, "strides", std::vector{}); - auto type = shape::parse_type(get(v, "type", "float")); - if(strides.empty()) - return shape{type, lens}; - else - return shape{type, lens, strides}; -} - -std::vector parser::parse_shapes(const value& v) const -{ - std::vector result; - std::transform( - v.begin(), v.end(), std::back_inserter(result), [&](auto&& x) { return parse_shape(x); }); - return result; -} - -void parser::load_settings(const value& v) -{ - if(v.contains("settings")) - settings = v.at("settings"); -} - -void parser::process(const value& v) -{ - if(not v.is_object()) - error("Input is not an object"); - parser p{}; - p.load_settings(v); - for(auto&& pp : v) - { - if(pp.get_key() == "settings") - continue; - get_action(pp.get_key())(p, pp.without_key()); - } -} - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/driver/precompile_op.cpp b/docker/rocm/migraphx/targets/gpu/driver/precompile_op.cpp deleted file mode 100644 index 2aec2a2d3..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/precompile_op.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -struct precompile_op : action -{ - static program create_preop_program(const operation& preop, std::vector inputs) - { - program p; - auto* mm = p.get_main_module(); - std::vector args; - inputs.pop_back(); - transform(inputs, range(inputs.size()), std::back_inserter(args), [&](auto input, auto i) { - return mm->add_parameter("x" + std::to_string(i), input); - }); - mm->add_instruction(preop, args); - return p; - } - - static operation get_code_object(const program& p) - { - MIGRAPHX_TIDY_CONST auto* mm = p.get_main_module(); - auto it = std::find_if(mm->begin(), mm->end(), [](const auto& ins) { - return (ins.name() == "gpu::code_object"); - }); - if(it == mm->end()) - MIGRAPHX_THROW("Failed to create code object"); - return it->get_operator(); - } - static void apply(const parser& p, const value& v) - { - context ctx; - auto inputs = p.parse_shapes(v.at("inputs")); - auto name = v.at("name").to(); - auto preop = make_op(name); - if(v.contains("fields")) - preop.from_value(v.at("fields")); - bool exhaustive = v.get("exhaustive", false); - auto prog = create_preop_program(preop, inputs); - run_passes(prog, {lowering{}, compile_ops{&ctx, exhaustive}}); - auto op = get_code_object(prog); - auto t = time_op(ctx, op, inputs, p.get(v, "iterations", 100)); - std::cout << preop << ": " << t << "ms" << std::endl; - } -}; - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/driver/run_op.cpp b/docker/rocm/migraphx/targets/gpu/driver/run_op.cpp deleted file mode 100644 index d5575a933..000000000 --- a/docker/rocm/migraphx/targets/gpu/driver/run_op.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace driver { - -struct run_op : action -{ - static void apply(const parser& p, const value& v) - { - context ctx; - auto inputs = p.parse_shapes(v.at("inputs")); - auto name = v.at("name").to(); - if(not contains(name, "::")) - name = "gpu::" + name; - auto op = make_op(name); - if(v.contains("fields")) - op.from_value(v.at("fields")); - auto t = time_op(ctx, op, inputs, p.get(v, "iterations", 100)); - std::cout << op << " -> " << op.compute_shape(inputs) << ": " << t << "ms" << std::endl; - } -}; - -} // namespace driver -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/fuse_ck.cpp b/docker/rocm/migraphx/targets/gpu/fuse_ck.cpp deleted file mode 100644 index bf9a269f3..000000000 --- a/docker/rocm/migraphx/targets/gpu/fuse_ck.cpp +++ /dev/null @@ -1,217 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct module; - -namespace gpu { - -struct ck_gemm -{ - operation op = make_op("dot"); - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op")); - } - - std::string name() const { return "gpu::ck_gemm"; } - - void check_gemm_shape(const shape& s) const - { - if(not contains(range(s.strides().rbegin(), s.strides().rbegin() + 3), 1)) - MIGRAPHX_THROW("Invalid shape for ck_gemm"); - } - - shape compute_shape(std::vector inputs, const std::vector& mods) const - { - check_shapes{inputs, *this}.same_ndims(); - if(inputs.size() < 2) - MIGRAPHX_THROW(name() + ": should have at least two inputs."); - auto a = inputs[0]; - auto b = inputs[1]; - for(const auto& input : inputs) - check_gemm_shape(input); - auto r = op.compute_shape({a, b}); - if(mods.empty()) - return r; - return r.with_type(mods.front()->get_output_shapes().front().type()); - } - - static bool is_ck_supported_type(shape::type_t t) - { - return contains({shape::half_type, shape::int8_type, shape::int32_type}, t); - } -}; -MIGRAPHX_REGISTER_OP(ck_gemm); - -struct ck_gemm_softmax_gemm : gemm_softmax_gemm -{ - std::string name() const { return "gpu::ck_gemm_softmax_gemm"; } -}; -MIGRAPHX_REGISTER_OP(ck_gemm_softmax_gemm); - -namespace { - -MIGRAPHX_PRED_MATCHER(is_ck_gemm, instruction_ref ins) -{ - if(ins->name() != "dot" and ins->name() != "quant_dot") - return false; - if(not ck_gemm::is_ck_supported_type(ins->get_shape().type())) - return false; - auto a = ins->inputs().front()->get_shape(); - auto b = ins->inputs().back()->get_shape(); - auto m = a.lens()[a.lens().size() - 2]; - auto n = b.lens().back(); - auto k = a.lens().back(); - auto batch_size = std::accumulate( - a.lens().rbegin() + 2, a.lens().rend(), std::size_t{1}, std::multiplies()); - // Integer gemms must be divisible by 4 in ck - if(contains({shape::int8_type, shape::int32_type}, ins->get_shape().type())) - { - if(m % 4 != 0) - return false; - if(n % 4 != 0) - return false; - if(k % 4 != 0) - return false; - } - auto device_name = trim(split_string(get_device_name(), ':').front()); - if(starts_with(device_name, "gfx94")) - { - if(ins->get_shape().type() == shape::half_type) - { - if(batch_size >= 64) - return m < 2048 or k <= 64 or n <= 384 or n >= 2048; - return true; - } - return true; - } - return k <= 2048; -} - -struct find_ck_gemm_pointwise -{ - // Find a gemm followed by a pointwise operation. - auto matcher() const - { - auto gemm = match::skip(match::name("contiguous"))( - match::name("dot", "quant_dot")(is_ck_gemm().bind("gemm"))); - return match::name("pointwise")(match::any_of[match::inputs()](gemm.bind("x"))); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto gemm_ins = r.instructions["gemm"]; - auto x_ins = r.instructions["x"]; // input after contiguous - auto* pm = ins->module_inputs().front(); - auto names = pm->get_parameter_names(); - std::sort(names.begin(), names.end()); - auto inputs = ins->inputs(); - auto gemm_it = std::find(inputs.begin(), inputs.end(), x_ins); - auto gemm_idx = gemm_it - inputs.begin(); - if(gemm_ins->get_shape().type() != shape::int32_type and - ins->get_shape().type() != gemm_ins->get_shape().type()) - return; - if(std::any_of(ins->inputs().begin(), ins->inputs().end(), [](auto input) { - return not ck_gemm::is_ck_supported_type(input->get_shape().type()); - })) - return; - if(std::any_of(ins->inputs().begin(), ins->inputs().end(), [](auto input) { - return not input->inputs().empty() and input->inputs().front()->name() == "capture"; - })) - return; - if(std::any_of(ins->inputs().begin(), ins->inputs().end(), [](auto input) { - return not input->inputs().empty() and input->inputs().front()->name() == "capture"; - })) - return; - assert(gemm_it != inputs.end()); - if(gemm_idx != 0) - { - auto first_param = pm->get_parameter(names[0]); - auto gemm_param = pm->get_parameter(names[gemm_idx]); - auto new_gemm_param = pm->add_parameter(names[0] + "_0", gemm_param->get_shape()); - auto new_first_param = - pm->add_parameter(names[gemm_idx] + "_0", first_param->get_shape()); - pm->replace_instruction(gemm_param, new_gemm_param); - pm->replace_instruction(first_param, new_first_param); - pm->remove_instruction(first_param); - pm->remove_instruction(gemm_param); - } - inputs.erase(gemm_it); - inputs.insert(inputs.begin(), gemm_ins->inputs().begin(), gemm_ins->inputs().end()); - - mpm.get_module().replace_instruction(ins, ck_gemm{gemm_ins->get_operator()}, inputs, {pm}); - } -}; - -struct find_ck_gemm -{ - auto matcher() const { return match::name("dot", "quant_dot")(is_ck_gemm().bind("gemm")); } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - mpm.get_module().replace_instruction(ins, ck_gemm{ins->get_operator()}, ins->inputs()); - } -}; - -struct find_ck_gemm_softmax_gemm -{ - auto matcher() const { return match::name("gpu::pre_gemm_softmax_gemm"); } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto v = ins->get_operator().to_value(); - assert(v.contains("scale")); - auto scale = v.at("scale").to(); - mpm.get_module().replace_instruction( - ins, ck_gemm_softmax_gemm{migraphx::make_op("dot"), scale}, ins->inputs()); - } -}; - -} // namespace - -void fuse_ck::apply(module_pass_manager& mpm) const -{ - match::find_matches(mpm, find_ck_gemm_softmax_gemm{}, find_ck_gemm_pointwise{}); - match::find_matches(mpm, find_ck_gemm{}); -} - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/fuse_mlir.cpp b/docker/rocm/migraphx/targets/gpu/fuse_mlir.cpp deleted file mode 100644 index 519955c21..000000000 --- a/docker/rocm/migraphx/targets/gpu/fuse_mlir.cpp +++ /dev/null @@ -1,1106 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_EXTRA_MLIR); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_MLIR_INPUT_FUSION); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_MLIR_REDUCE_FUSION); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_MLIR); -/** - * @brief Declares a new MIGraphX environment variable which forces to generate - * only specific MLIR operations. - * - * The variable, if defined, forces MIGraphX to use only specific operations - * with MLIR regardless of the underlying GPU architecture. The variable accepts - * a list of operations separated by comma. The variable recognizes the following - * operations: "fused", "convolution", "dot". If the variable is not defined MIGraphX - * will decide by itself which operations to delegate to MLIR. The variable is - * intended to be primarily used by rocMLIR developers. - */ -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_USE_SPECIFIC_OPS); - -bool mlir_enabled() -{ -#ifdef MIGRAPHX_MLIR - const bool mlir_disabled = enabled(MIGRAPHX_DISABLE_MLIR{}); - return not mlir_disabled; -#else - return false; -#endif -} - -namespace { -struct requested -{ -}; -struct rejected -{ -}; -} // namespace - -static bool is_negated_op(const std::string& s) -{ - if(s.empty()) - return false; - return contains({'!', '~'}, s[0]); -} - -template -static std::vector get_usage() -{ - static const auto options = - split_string(string_value_of(MIGRAPHX_MLIR_USE_SPECIFIC_OPS{}, ""), ','); - static const bool enabled = std::is_same{}; - std::vector result; - auto remove_not_symbol = [&](const std::string& s) { - if(is_negated_op(s)) - return s.substr(1); - return s; - }; - transform_if( - options.begin(), - options.end(), - std::back_inserter(result), - [&](const std::string& option) { - if(option.empty()) - return false; - if(is_negated_op(option)) - return not enabled; - return enabled; - }, - remove_not_symbol); - return result; -} - -template -static bool specific_op(std::string_view option, bool fallback = false) -{ - static const auto options = get_usage(); - if(options.empty()) - return fallback; - if(contains(option, "fused") and contains(options, "fused")) - return true; - return contains(options, option); -} - -bool mlir_attention_enabled(context* ctx) -{ -#ifdef MIGRAPHX_MLIR - if(not mlir_enabled()) - return false; - if(specific_op("attention")) - return false; - // Enable attention by default for mi300 - if(ctx != nullptr and starts_with(ctx->get_current_device().get_gfx_name(), "gfx94")) - return true; - return specific_op("attention"); -#else - return false; -#endif -} - -#ifdef MIGRAPHX_MLIR - -struct mlir_op -{ - std::string name() const { return "gpu::mlir_op"; } - operation op = make_op("convolution"); - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op")); - } - - // Check if the shape can be created from a transpose/broadcast/slice - static bool is_mlir_compatible(const shape& s) - { - if(s.standard() or s.packed() or s.scalar() or s.ndim() == 1) - return true; - auto ns = reorder_shape(s, find_permutation(s)); - std::vector stride_ratios; - auto last = std::find(ns.strides().begin(), ns.strides().end(), 0); - if(*std::prev(last) != 1) - return false; - std::adjacent_difference(ns.strides().begin(), - last, - std::back_inserter(stride_ratios), - [](auto y, auto x) -> std::size_t { - assert(y != 0); - if((x % y) != 0) - return 0; - return x / y; - }); - return std::equal(stride_ratios.begin() + 1, - stride_ratios.end(), - ns.lens().begin() + 1, - [](auto ratio, auto len) { return ratio >= len; }); - } - - shape compute_shape(const std::vector& inputs, const std::vector& mods) const - { - module_ref mod = mods[0]; - check_shapes{inputs, *this}.has_at_least(1); - if(mods.size() != 1) - MIGRAPHX_THROW("should have one submodule."); - - if(not std::all_of(inputs.begin(), inputs.end(), &is_mlir_compatible)) - MIGRAPHX_THROW("Shape is not mlir compatible."); - - auto result = - mod->compute_shapes(inputs, {.name = name(), .strict_type = true, .strict_lens = true}); - if(result.size() == 1) - return result.front(); - return shape{result}; - } -}; -MIGRAPHX_REGISTER_OP(mlir_op); - -namespace { - -const auto& reshaper_names() -{ - // clang-format off - static const std::unordered_set names = { - "slice", - "transpose", - "multibroadcast", - "broadcast", - "contiguous", - "reshape", - "lazy_reshape", - "squeeze", - "flatten", - "unsqueeze" - }; - // clang-format on - return names; -} - -std::tuple> -get_fusable_input_op_stream(instruction_ref lower_input) -{ - instruction_ref upper_input = lower_input; - std::vector op_stream; - while(contains(reshaper_names(), upper_input->name())) - { - operation op = upper_input->get_operator(); - op_stream.push_back(op); - upper_input = upper_input->inputs().at(0); - } - return {upper_input, op_stream}; -} - -void fuse_input_ops(module_ref mm, - const std::vector& inputs, - std::unordered_map* map_ins) -{ - assert(map_ins != nullptr); - size_t input_cnt = mm->get_parameters().size(); - for(instruction_ref input : inputs) - { - if(contains(*map_ins, input)) - continue; - auto [upper_input, op_stream] = get_fusable_input_op_stream(input); - if(not contains(*map_ins, upper_input)) - (*map_ins)[upper_input] = - mm->add_parameter(param_name(input_cnt++), upper_input->get_shape().as_standard()); - instruction_ref prev_input = (*map_ins)[upper_input]; - for(const auto& op : reverse(op_stream)) - { - prev_input = mm->add_instruction(op, {prev_input}); - } - (*map_ins)[input] = prev_input; - } -} - -std::tuple> -fuse_input_ops_and_gemm_based_op(module_ref mm, - const std::vector& gemm_based_op_inputs, - const operation& gemm_based_op) -{ - std::vector top_inputs; - std::vector imm_inputs; - size_t input_cnt = 0; - for(instruction_ref input : gemm_based_op_inputs) - { - auto [upper_input, op_stream] = get_fusable_input_op_stream(input); - top_inputs.push_back(upper_input); - instruction_ref prev_input = - mm->add_parameter(param_name(input_cnt++, "y"), upper_input->get_shape().as_standard()); - for(const auto& op : reverse(op_stream)) - { - prev_input = mm->add_instruction(op, {prev_input}); - } - imm_inputs.push_back(prev_input); - } - instruction_ref new_gemm_based_op = mm->add_instruction(gemm_based_op, imm_inputs); - return {new_gemm_based_op, top_inputs}; -} - -enum class mlir_mode -{ - all, - fast, - int8, - none -}; - -auto is_mlir_dot(mlir_mode mode) -{ - return match::make_basic_pred_matcher([=](instruction_ref ins) { - if(mode == mlir_mode::none) - return false; - if(ins->name() != "dot" and ins->name() != "quant_dot") - return false; - // dot operation where (FP8 * FP8 = FP8) is not available in MLIR. rocBLAS/hipBLASLt should - // have the support for it. - if(contains(fp8_types{}.get(), ins->get_shape().type())) - return false; - if(mode != mlir_mode::fast) - return true; - auto a = ins->inputs().front()->get_shape(); - auto b = ins->inputs().back()->get_shape(); - // auto m = a.lens()[a.lens().size() - 2]; - // auto n = b.lens().back(); - auto k = a.lens().back(); - // Skipping GEMMs with a K dimension greater than 2048 is a course-grained strategy - // to avoid poor-performing GEMM kernels from MLIR - // To-do: Investigate a more precise strategy - return k <= 1024; - }); -} - -auto is_mlir_conv(mlir_mode mode) -{ - return match::make_basic_pred_matcher([=](instruction_ref ins) { - if(mode == mlir_mode::none) - return false; - if(ins->name() != "convolution" and ins->name() != "quant_convolution") - return false; - auto input = ins->inputs().front()->get_shape(); - value v = ins->get_operator().to_value(); - auto group = v.at("group").to(); - // Avoid MLIR assertion: Index < Length && "Invalid index!" - if(ins->get_shape().lens().size() != 4 and group > 1) - return false; - std::set supported_types = fp8_types{}.get(); - supported_types.insert(shape::int8_type); - if(contains(supported_types, input.type())) - return true; - if(mode == mlir_mode::all) - return true; - // No winograd for group convolution - if(group > 1) - return true; - auto w = ins->inputs().at(1)->get_shape(); - if(w.lens().size() != 4) - return true; - if(w.lens()[2] != w.lens()[3]) - return true; - return (w.lens()[3] % 3) != 0; - }); -} - -std::unordered_map -create_param_map_with_literals(module_ref mm, const module* pm, const shape& shape) -{ - std::unordered_map ins_map; - for(auto ins : iterator_for(*pm)) - { - if(ins->name() != "@literal") - { - continue; - } - literal r = ins->get_literal(); - instruction_ref literal = mm->add_literal(r); - instruction_ref mbcast = - mm->add_instruction(make_op("multibroadcast", {{"out_lens", shape.lens()}}), literal); - ins_map[ins] = mbcast; - } - return ins_map; -} - -instruction_ref unroll_pointwise(module& main_mod, - instruction_ref pos, - const operation& op, - const std::vector& inputs, - const std::vector& mod_args) -{ - if(op.name() == "pointwise") - { - auto* sub_pm = mod_args.front(); - auto param_map_2 = create_param_map_with_literals( - &main_mod, sub_pm, op.compute_shape(to_shapes(inputs), mod_args)); - return main_mod.insert_inline(pos, *sub_pm, inputs, ¶m_map_2) - .front(); // cppcheck-suppress returnDanglingLifetime; - } - return main_mod.insert_instruction(pos, op, inputs, mod_args); -} - -// Whitelist supported fusion options, including imposing type constraints -// for cases where MLIR only supports an operation (usually a pointwise function) -// on particular types. -bool is_pointwise_op_supported_by_mlir(const instruction& i) -{ - using type_t = shape::type_t; - const auto& name = i.name(); - const auto result_type = i.get_shape().type(); - const std::initializer_list allowed_types = {type_t::float_type, - type_t::bf16_type, - type_t::half_type, - type_t::fp8e4m3fnuz_type, - type_t::fp8e5m2fnuz_type, - type_t::fp8e4m3fn_type, - type_t::fp8e5m2_type, - type_t::int8_type, - type_t::uint8_type, - type_t::int32_type, - type_t::uint32_type, - type_t::bool_type}; - // Preliminary type check. - if(not contains(allowed_types, result_type)) - { - return false; - } - const std::initializer_list any_type_ops = {"@literal", "@param", "@return"}; - const std::initializer_list no_bool_ops = { - "convolution", - "quant_convolution", - "dot", - "quant_dot", - "add", - "clip", - "relu", - "sub", - "mul", - "div", - "pow", - "where", - "quantizelinear", - "dequantizelinear", - "abs", - "neg", - }; - const std::initializer_list fp_only_ops = { - "ceil", - "erf", - "exp", - "floor", - "log", - "recip", - "sqrt", - "rsqrt", - "sigmoid", - "softmax", - "tanh", - }; - std::set float_types = {type_t::float_type, - type_t::half_type, - type_t::bf16_type, - type_t::fp8e4m3fnuz_type, - type_t::fp8e5m2fnuz_type, - type_t::fp8e4m3fn_type, - type_t::fp8e5m2_type}; - bool is_float = contains(float_types, result_type); - if(contains(any_type_ops, name)) - return true; - if(result_type != type_t::bool_type and contains(no_bool_ops, name)) - return true; - if(is_float and contains(fp_only_ops, name)) - return true; - // Only conversions between floating types are known to be unambigiously - // supported. - if(is_float and name == "convert") - { - if(contains(fp8_types{}.get(), result_type)) - { - return false; - } // else - return std::all_of(i.inputs().begin(), i.inputs().end(), [](const auto& arg) { - return contains({type_t::float_type, type_t::half_type, type_t::bf16_type}, - arg->get_shape().type()); - }); - } - return false; -} - -bool is_reduce_op_supported_by_mlir(const instruction& i) -{ - using type_t = shape::type_t; - const auto& name = i.name(); - const auto result_type = i.get_shape().type(); - const std::initializer_list allowed_types = {type_t::float_type, - type_t::half_type, - type_t::bf16_type, - type_t::fp8e4m3fnuz_type, - type_t::fp8e5m2fnuz_type, - type_t::fp8e4m3fn_type, - type_t::fp8e5m2_type}; - - // Preliminary type check. - if(not contains(allowed_types, result_type)) - { - return false; - } - const std::initializer_list reduce_ops = {"reduce_mean", "reduce_sum"}; - return contains(reduce_ops, i.name()); -} - -// A separate function so we can remove operators that are supported by mlir -// but not supported for an input fusion. -bool is_pointwise_op_supported_by_mlir_for_input(const instruction& i) -{ - return is_pointwise_op_supported_by_mlir(i); -} - -MIGRAPHX_PRED_MATCHER(mlir_split_reduce, instruction_ref ins) -{ - if(ins->name() != "split_fused_reduce") - return false; - auto* mod_arg = ins->module_inputs().front(); - auto supported_reshapes = reshaper_names(); - supported_reshapes.erase("slice"); - std::unordered_set builtins = {"@param", "@literal", "@return"}; - for(const auto i : iterator_for(*mod_arg)) - { - if(is_reduce(*i)) - { - if(not is_reduce_op_supported_by_mlir(*i)) - return false; - } - else if(i->name() == "pointwise") - { - if(not std::all_of(i->module_inputs().front()->begin(), - i->module_inputs().front()->end(), - &is_pointwise_op_supported_by_mlir)) - return false; - } - else if(not contains(reshaper_names(), i->name()) and not contains(builtins, i->name())) - { - return false; - } - } - return true; -} - -MIGRAPHX_PRED_MATCHER(mlir_pointwise, instruction_ref ins) -{ - if(ins->name() != "pointwise") - return false; - auto* pm = ins->module_inputs().front(); - return std::all_of(pm->begin(), pm->end(), &is_pointwise_op_supported_by_mlir); -} - -MIGRAPHX_PRED_MATCHER(mlir_input_pointwise, instruction_ref ins) -{ - if(ins->name() != "pointwise") - return false; - auto* pm = ins->module_inputs().front(); - return std::all_of(pm->begin(), pm->end(), &is_pointwise_op_supported_by_mlir_for_input); -} - -std::vector mlir_contiguous(module_pass_manager& mpm, - const std::vector& inputs) -{ - std::vector result; - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(result), [&](instruction_ref input) { - if(input->get_shape().packed() or input->get_shape().broadcasted()) - return input; - return mpm.get_module().insert_instruction( - std::next(input), make_op("contiguous"), input); - }); - return result; -} - -struct find_mlir_split_reduce -{ - mlir_mode conv_mode = mlir_mode::none; - mlir_mode dot_mode = mlir_mode::none; - auto matcher() const - { - auto dot_or_conv = match::name("gpu::mlir_op"); - // TODO: Handle reshapes inbetween - return mlir_split_reduce()(match::any_of[match::inputs()](dot_or_conv.bind("gemm"))); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto reduce_ins = r.result; - auto gemm_ins = r.instructions["gemm"]; - assert(gemm_ins->get_shape().sub_shapes().empty()); - auto* rm = reduce_ins->module_inputs().front(); - auto names = rm->get_parameter_names(); - std::sort(names.begin(), names.end()); - module_ref gemm_old_mm = gemm_ins->module_inputs().front(); - module_ref mm = mpm.create_module(gemm_old_mm->name() + "_" + rm->name(), *gemm_old_mm); - // remove last return instruction - if(std::prev(mm->end())->name() == "@return") - { - mm->remove_instruction(std::prev(mm->end())); - } - mm->set_bypass(); - std::unordered_map param_map; - param_map[gemm_ins] = std::prev(mm->end()); - bool gemm_has_multi_outs = gemm_ins->outputs().size() > 1; - auto return_vals = mm->fuse(*rm, reduce_ins->inputs(), ¶m_map, &unroll_pointwise); - if(gemm_has_multi_outs) - { - return_vals.insert(return_vals.end(), param_map[gemm_ins]); - } - mm->add_return(return_vals); - std::vector inputs; - std::copy_if(reduce_ins->inputs().begin(), - reduce_ins->inputs().end(), - std::back_inserter(inputs), - [&](auto input) { return input != gemm_ins; }); - inputs.insert(inputs.end(), gemm_ins->inputs().begin(), gemm_ins->inputs().end()); - if(gemm_has_multi_outs) - { - auto fused_ins = mpm.get_module().insert_instruction( - reduce_ins, mlir_op{gemm_ins->get_operator()}, mlir_contiguous(mpm, inputs), {mm}); - auto dot_ins = mpm.get_module().insert_instruction( - reduce_ins, - migraphx::make_op("get_tuple_elem", {{"index", return_vals.size() - 1}}), - fused_ins); - - mpm.get_module().replace_instruction(gemm_ins, dot_ins); - for(const auto& outs : reduce_ins->outputs()) - { - assert(outs->get_operator().name() == "get_tuple_elem"); - mpm.get_module().replace_instruction(outs, outs->get_operator(), fused_ins); - } - } - else - { - mpm.get_module().replace_instruction( - reduce_ins, mlir_op{gemm_ins->get_operator()}, mlir_contiguous(mpm, inputs), {mm}); - } - } -}; - -struct find_mlir_fused_ops -{ - mlir_mode conv_mode = mlir_mode::none; - mlir_mode dot_mode = mlir_mode::none; - auto matcher() const - { - auto reshapes = reshaper_names(); - // slice is not supported - reshapes.erase("slice"); - auto dot_or_conv = match::skip(match::name(reshapes))( - match::any_of(is_mlir_dot(dot_mode), is_mlir_conv(conv_mode)).bind("gemm_based_op")); - return mlir_pointwise()(match::any_of[match::inputs()](dot_or_conv.bind("x"))); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto pw_ins = r.result; - auto gemm_based_op = r.instructions["gemm_based_op"]; - auto x_ins = r.instructions["x"]; // input to pointwise after reshaper op stream - auto* pm = pw_ins->module_inputs().front(); - auto pw_inputs = pw_ins->inputs(); - // only of one of the inputs to pointwise module should be dependent on conv/gemm that is - // being fused, otherwise it can create invalid graph transformation - if(std::any_of(pw_inputs.begin(), pw_inputs.end(), [&](const auto& i) { - return i != x_ins and reaches(gemm_based_op, i); - })) - return; - auto names = pm->get_parameter_names(); - std::sort(names.begin(), names.end()); - module_ref mm = mpm.create_module("mlir_" + pm->name()); - mm->set_bypass(); - auto [anchor_op, top_inputs] = fuse_input_ops_and_gemm_based_op( - mm, gemm_based_op->inputs(), gemm_based_op->get_operator()); - std::unordered_map param_map = - create_param_map_with_literals(mm, pm, pw_ins->get_shape()); - auto [upper_input, op_stream] = get_fusable_input_op_stream(x_ins); - assert(upper_input == gemm_based_op); - auto prev_input = anchor_op; - for(const auto& op : reverse(op_stream)) - { - prev_input = mm->add_instruction(op, {prev_input}); - } - assert(prev_input->get_shape().lens() == x_ins->get_shape().lens()); - param_map[x_ins] = prev_input; // this is to avoid adding parameter for gemm/conv reshaped - // input to pointwise in new fused module - bool gemm_has_multi_outs = gemm_based_op->outputs().size() > 1; - auto reshaped_gemm = x_ins; - std::vector reshapes_vec; - while(reshaped_gemm != gemm_based_op) - { - reshapes_vec.push_back(reshaped_gemm); - gemm_has_multi_outs = gemm_has_multi_outs or reshaped_gemm->outputs().size() > 1; - reshaped_gemm = reshaped_gemm->inputs().at(0); - } - reshapes_vec.push_back(reshaped_gemm); - - auto return_vals = mm->fuse(*pm, pw_ins->inputs(), ¶m_map); - if(gemm_has_multi_outs) - { - return_vals.insert(return_vals.begin(), anchor_op); - } - mm->add_return(return_vals); - - std::vector inputs; - std::copy_if(pw_ins->inputs().begin(), - pw_ins->inputs().end(), - std::back_inserter(inputs), - [&](auto input) { return input != x_ins; }); - inputs.insert(inputs.end(), top_inputs.begin(), top_inputs.end()); - if(gemm_has_multi_outs) - { - auto fused_ins = mpm.get_module().insert_instruction( - pw_ins, mlir_op{gemm_based_op->get_operator()}, mlir_contiguous(mpm, inputs), {mm}); - mpm.get_module().replace_instruction( - pw_ins, migraphx::make_op("get_tuple_elem", {{"index", 1}}), fused_ins); - auto dot_ins = mpm.get_module().insert_instruction( - pw_ins, migraphx::make_op("get_tuple_elem", {{"index", 0}}), fused_ins); - // move all the reshape instructions and original GEMM instruction after the fused op to - // avoid generating invalid migraphx program - for(const auto& orig_i : reverse(reshapes_vec)) - { - mpm.get_module().move_instruction(orig_i, pw_ins); - } - mpm.get_module().replace_instruction(gemm_based_op, dot_ins); - } - else - { - mpm.get_module().replace_instruction( - pw_ins, mlir_op{gemm_based_op->get_operator()}, mlir_contiguous(mpm, inputs), {mm}); - } - } -}; - -template -struct find_mlir_standalone_op -{ - mlir_mode mode = mlir_mode::none; - std::size_t* counter = nullptr; - auto matcher() const { return Matcher(mode); } - - std::string get_count() const - { - if(counter == nullptr) - MIGRAPHX_THROW("Invalid counter"); - return std::to_string((*counter)++); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto gemm_based_op = r.result; - // enable only for fp32/fp16/i8/fp8 types - if(std::any_of(gemm_based_op->inputs().begin(), gemm_based_op->inputs().end(), [&](auto i) { - return not contains({shape::type_t::float_type, - shape::type_t::half_type, - shape::type_t::bf16_type, - shape::type_t::int8_type, - shape::type_t::fp8e4m3fnuz_type, - shape::type_t::fp8e5m2fnuz_type, - shape::type_t::fp8e4m3fn_type, - shape::type_t::fp8e5m2_type}, - i->get_shape().type()); - })) - return; - std::string module_name = "mlir_" + gemm_based_op->name() + get_count(); - if(mpm.get_module().name() != "main") - module_name = mpm.get_module().name() + ":" + module_name; - module_ref mm = mpm.create_module(module_name); - mm->set_bypass(); - auto [anchor_op, top_inputs] = fuse_input_ops_and_gemm_based_op( - mm, gemm_based_op->inputs(), gemm_based_op->get_operator()); - mm->add_return({anchor_op}); - mpm.get_module().replace_instruction(gemm_based_op, - mlir_op{gemm_based_op->get_operator()}, - mlir_contiguous(mpm, top_inputs), - {mm}); - } -}; - -using find_mlir_standalone_convolution_op = find_mlir_standalone_op<&is_mlir_conv>; -using find_mlir_standalone_dot_op = find_mlir_standalone_op<&is_mlir_dot>; - -struct find_mlir_standalone_attention_op -{ - mlir_mode dot_mode = mlir_mode::none; - - auto matcher() const - { - auto gemm1 = - match::skip(match::name("contiguous"))(match::used_once(), is_mlir_dot(dot_mode)) - .bind("gemm1"); - auto fused_reduce = - match::name("fused_reduce")(match::used_once(), - match::any_of[match::inputs()]( - match::skip(match::name("reshape").bind("rsp"))(gemm1))) - .bind("fused_reduce"); - return is_mlir_dot(dot_mode)(match::arg(0)(fused_reduce)).bind("gemm2"); - } - - std::unordered_map - invert_map_ins(const std::unordered_map& map_ins) const - { - std::unordered_map inverse_map; - for(auto const& [key, value] : map_ins) - { - assert(not contains(inverse_map, value)); - inverse_map[value] = key; - } - return inverse_map; - } - - auto finalize_attention_module(module_ref m) const - { - eliminate_common_subexpression{}.apply(*m); - dead_code_elimination{}.apply(*m); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto gemm2 = r.instructions["gemm2"]; - auto fused_reduce = r.instructions["fused_reduce"]; - auto gemm1 = r.instructions["gemm1"]; - - auto axes = fused_reduce->get_operator().to_value()["axes"]; - if(axes.size() != 1) - return; - - module m_attn; - std::unordered_map map_main_to_mattn; - - // Add first gemm and fuse any input shape ops - module fuse_gemm1; - auto [anchor_op, top_inputs] = - fuse_input_ops_and_gemm_based_op(&fuse_gemm1, gemm1->inputs(), gemm1->get_operator()); - fuse_gemm1.add_return({anchor_op}); - m_attn.add_params(top_inputs, &map_main_to_mattn); - - std::unordered_map map_gemm1_to_mattn(map_main_to_mattn); - auto m_gemm1 = m_attn.fuse(fuse_gemm1, top_inputs, &map_gemm1_to_mattn).front(); - map_main_to_mattn[gemm1] = m_gemm1; - - if(contains(r.instructions, "rsp")) - { - auto rsp = r.instructions["rsp"]; - auto m_rsp = m_attn.add_instruction(rsp->get_operator(), {m_gemm1}); - map_main_to_mattn[rsp] = m_rsp; - } - // Add pointwise-softmax, unroll any pointwise modules back to base ops - m_attn.add_params(fused_reduce->inputs(), &map_main_to_mattn); - std::unordered_map map_mfr_to_mattn(map_main_to_mattn); - auto pw_softmax = m_attn - .fuse(*fused_reduce->module_inputs().front(), - fused_reduce->inputs(), - &map_mfr_to_mattn, - &unroll_pointwise) - .front(); - - // fused_reduce submodule should end with a softmax - auto result = match::match_instruction(m_attn, pw_softmax, match::softmax()); - if(result.result != pw_softmax) - return; - - // Insert explict softmax op - required for MLIR - auto softmax_in = result.instructions["x"]; - auto softmax = m_attn.insert_instruction( - std::next(softmax_in), make_op("softmax", {{"axis", axes.front()}}), softmax_in); - map_main_to_mattn[fused_reduce] = softmax; - - // all preceeding ops should be fusable ops - if(not std::all_of(m_gemm1, softmax, [](auto i) { - return (is_pointwise_op_supported_by_mlir(i) or - contains(reshaper_names(), i.name())); - })) - return; - - // Add second gemm and fuse any input shape ops - module fuse_gemm2; - auto [anchor_op2, top_inputs2] = - fuse_input_ops_and_gemm_based_op(&fuse_gemm2, gemm2->inputs(), gemm2->get_operator()); - fuse_gemm2.add_return({anchor_op2}); - m_attn.add_params(top_inputs2, &map_main_to_mattn); - - std::unordered_map map_gemm2_to_mattn(map_main_to_mattn); - auto m_gemm2 = m_attn.fuse(fuse_gemm2, top_inputs2, &map_gemm2_to_mattn).front(); - map_main_to_mattn[gemm2] = m_gemm2; - - // Fuse any succeeding pointwise module - if(contains(r.instructions, "trailing_pm")) - { - auto trailing_pm_ins = r.instructions["trailing_pm"]; - auto lit_map = create_param_map_with_literals( - &m_attn, trailing_pm_ins->module_inputs().front(), trailing_pm_ins->get_shape()); - m_attn.add_params(trailing_pm_ins->inputs(), &map_main_to_mattn); - map_main_to_mattn.insert(lit_map.begin(), lit_map.end()); - std::unordered_map map_pm_to_mattn(map_main_to_mattn); - auto fused_pw_outs = m_attn - .fuse(*trailing_pm_ins->module_inputs().front(), - trailing_pm_ins->inputs(), - &map_pm_to_mattn) - .front(); - map_main_to_mattn[trailing_pm_ins] = fused_pw_outs; - m_attn.add_return({fused_pw_outs}); - } - else - { - m_attn.add_return({m_gemm2}); - } - - finalize_attention_module(&m_attn); - auto map_mattn_to_main = invert_map_ins(map_main_to_mattn); - auto new_inputs = m_attn.get_inputs(map_mattn_to_main); - - module_ref mpm_attn = mpm.create_module( - "mlir_attn_" + fused_reduce->module_inputs().front()->name(), std::move(m_attn)); - mpm_attn->set_bypass(); - - mpm.get_module().replace_instruction( - r.result, mlir_op{gemm1->get_operator()}, mlir_contiguous(mpm, new_inputs), {mpm_attn}); - } -}; - -struct find_mlir_attention_fused_ops : public find_mlir_standalone_attention_op -{ - auto matcher() const - { - auto standalone_matcher = find_mlir_standalone_attention_op::matcher(); - return mlir_pointwise()( - match::any_of[match::inputs()](standalone_matcher).bind("trailing_pm")); - ; - } -}; - -struct find_pointwise_mlir -{ - auto supported_pointwise() const { return mlir_input_pointwise(match::used_once()); } - - auto matcher() const - { - return match::name("gpu::mlir_op")(match::any_of[match::inputs()](supported_pointwise())); - } - - static bool is_simple_op(const_module_ref pm, std::initializer_list op_names) - { - auto last = std::prev(pm->end()); - assert(last->name() == "@return"); - if(last->inputs().size() != 1) - return false; - auto rins = last->inputs().front(); - auto op_ins = std::find_if(pm->begin(), pm->end(), [](const instruction& x) { - return not contains({"@param", "@literal", "broadcast", "multibroadcast"}, x.name()); - }); - if(op_ins != rins) - return false; - return contains(op_names, op_ins->name()); - } - - static instruction_ref insert_pointwise(module& m, - instruction_ref ins, - const operation& op, - const std::vector& inputs, - const std::vector& mod_args) - { - // Only used in assert - (void)mod_args; - assert(mod_args.empty()); - return insert_common_op(m, ins, op, inputs, {.common_type = false}); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - - auto* mm = ins->module_inputs().front(); - std::vector pws; - std::copy_if( - ins->inputs().begin(), - ins->inputs().end(), - std::back_inserter(pws), - [&](instruction_ref input) { - if(not match::instruction_matches(mpm.get_module(), input, supported_pointwise())) - return false; - auto* pm = input->module_inputs().front(); - if(input->inputs().size() > 1 and not is_simple_op(pm, {"dequantizelinear"})) - { - if(not enabled(MIGRAPHX_ENABLE_MLIR_INPUT_FUSION{})) - return false; - } - return true; - }); - if(pws.empty()) - return; - - std::string module_name; - std::transform( - pws.begin(), pws.end(), join_back_inserter(module_name), [](instruction_ref pw) { - return pw->module_inputs().front()->name() + ":"; - }); - module_name += mm->name(); - module_ref m = mpm.create_module(module_name); - m->set_bypass(); - - std::unordered_map map_ins; - for(auto pw : pws) - { - auto* pm = pw->module_inputs().front(); - fuse_input_ops(m, pw->inputs(), &map_ins); - auto rins = m->fuse(*pm, pw->inputs(), &map_ins, &insert_pointwise).front(); - map_ins[pw] = rins; - } - - auto ret = m->fuse(*mm, ins->inputs(), &map_ins); - m->add_return({ret}); - - auto inputs = find_inputs(map_ins, &mpm.get_module(), m); - mpm.get_module().replace_instruction( - ins, ins->get_operator(), mlir_contiguous(mpm, inputs), {m}); - } -}; - -struct find_unpack_int4_mlir_op -{ - auto matcher() const - { - return match::name("gpu::mlir_op")( - match::any_of[match::inputs()](match::name("unpack_int4").bind("unpack_int4"))); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto* mm = ins->module_inputs().front(); - module_ref nm = mpm.create_module("int4:" + mm->name()); - nm->set_bypass(); - - std::vector x_in; - std::unordered_map map_ins; - int ct = 0; - - for(auto input : ins->inputs()) - { - if(input->get_operator().name() == "unpack_int4") - { - auto unpack_input = input->inputs()[0]; - instruction_ref t_ins = - nm->add_parameter(param_name(++ct), unpack_input->get_shape().as_standard()); - map_ins[input] = nm->add_instruction(input->get_operator(), t_ins); - x_in.push_back(unpack_input); - } - else - { - map_ins[input] = - nm->add_parameter(param_name(++ct), input->get_shape().as_standard()); - x_in.push_back(input); - } - } - auto ret = nm->fuse(*mm, ins->inputs(), &map_ins); - nm->add_return({ret}); - mpm.get_module().replace_instruction(ins, ins->get_operator(), x_in, {nm}); - } -}; - -} // namespace - -#endif // MIGRAPHX_MLIR - -void fuse_mlir::apply(module_pass_manager& mpm) const -{ -#ifdef MIGRAPHX_MLIR - std::size_t counter = 0; - const auto& device_name = ctx == nullptr ? "" : ctx->get_current_device().get_gfx_name(); - const bool is_navi = starts_with(device_name, "gfx11") or starts_with(device_name, "gfx12"); - - auto get_mode = [&](std::string_view option, mlir_mode m1, mlir_mode m2 = mlir_mode::fast) { - if(specific_op(option)) - return mlir_mode::none; - if(specific_op(option)) - return mlir_mode::all; - if(is_navi) - return mlir_mode::all; - return std::max(m1, m2); - }; - - // Attention offloads; default disabled - if(mlir_attention_enabled(ctx) or enable_extra) - { - match::find_matches(mpm, find_mlir_attention_fused_ops{mlir_mode::all}); - mpm.run_pass(dead_code_elimination{}); - match::find_matches(mpm, find_mlir_standalone_attention_op{mlir_mode::all}); - mpm.run_pass(dead_code_elimination{}); - } - - match::find_matches( - mpm, - find_mlir_fused_ops{.conv_mode = get_mode("fused_convolution", mlir_mode::fast), - .dot_mode = get_mode("fused_dot", mlir_mode::fast)}); - - match::find_matches( - mpm, - find_mlir_standalone_convolution_op{.mode = get_mode("convolution", mlir_mode::fast), - .counter = &counter}, - find_mlir_standalone_dot_op{.mode = get_mode("dot", mlir_mode::fast), .counter = &counter}); - - mpm.run_pass(dead_code_elimination{}); - if(enabled(MIGRAPHX_ENABLE_MLIR_REDUCE_FUSION{})) - { - match::find_matches( - mpm, - find_mlir_split_reduce{.conv_mode = get_mode("fused_convolution", mlir_mode::fast), - .dot_mode = get_mode("fused_dot", mlir_mode::fast)}); - } - - match::find_matches(mpm, find_pointwise_mlir{}); - match::find_matches(mpm, find_unpack_int4_mlir_op{}); - -#else - (void)mpm; -#endif -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/fuse_ops.cpp b/docker/rocm/migraphx/targets/gpu/fuse_ops.cpp deleted file mode 100644 index 5e93ccf5e..000000000 --- a/docker/rocm/migraphx/targets/gpu/fuse_ops.cpp +++ /dev/null @@ -1,1060 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_HIPBLASLT_GEMM) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_MIOPEN_FUSION) -#if MIGRAPHX_USE_MIOPEN -struct fusion -{ - using op_t = miopenFusionOpDescriptor_t; - shared fp; - - // Used as a temporary hack to keep descriptor references alive - std::vector> storage; - - template - auto keep_alive(T x) - { - auto result = share(std::move(x)); - storage.push_back(result); - return result; - } - - fusion() = default; - - fusion(const shape& input) - { - assert(input.standard()); - auto t = make_tensor(input); - fp = make_fusion_plan(t); - assert(fp); - keep_alive(std::move(t)); - } - - bool empty() const { return fp == nullptr; } - - op_t operator[](std::size_t i) const - { - assert(fp); - op_t result; - auto status = miopenFusionPlanGetOp(fp.get(), i, &result); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("Failed retrieving operator at " + std::to_string(i)); - return result; - } - - auto get() const - { - assert(fp); - return fp.get(); - } - - op_t create_bias(const shape& bias) - { - assert(fp); - op_t result; - auto b = shape{bias.type(), {1, bias.lens().at(1), 1, 1}}; - auto t = keep_alive(make_tensor(b)); - auto status = miopenCreateOpBiasForward(fp.get(), &result, t.get()); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("Creating operator failed"); - return result; - } - - op_t create_relu() - { - assert(fp); - op_t result; - auto status = miopenCreateOpActivationForward(fp.get(), &result, miopenActivationRELU); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("Creating operator failed"); - return result; - } - - op_t create_conv(const op::convolution& op, const shape& weights) - { - assert(fp); - op_t result; - auto cd = keep_alive(make_conv(op)); - auto t = keep_alive(make_tensor(weights)); - auto status = miopenCreateOpConvForward(fp.get(), &result, cd.get(), t.get()); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("Creating operator failed"); - return result; - } - - shape get_workspace(context&) - { - // assert(fp); - // TODO: Use zero workspace for now - std::size_t ws_size = 0; - // int algo_count = 1; - // miopenConvFwdAlgorithm_t algo; - // miopenFusionPlanConvolutionGetAlgo(fp.get(), 1, &algo_count, &algo); - // miopenFusionPlanGetWorkSpaceSize(ctx.get_stream().get_miopen(), fp.get(), &ws_size, - // algo); - return shape{shape::int8_type, {ws_size}}; - } - - bool compile(context& ctx) - { - assert(fp); - return miopenCompileFusionPlan(ctx.get_stream().get_miopen(), fp.get()) == - miopenStatusSuccess; - } - - argument execute(context& ctx, - const fused_operator_args& fargs, - const argument& x, - const argument& y) const - { - assert(fp); - auto x_td = make_tensor(x.get_shape()); - auto y_td = make_tensor(y.get_shape()); - auto status = miopenExecuteFusionPlan(ctx.get_stream().get_miopen(), - fp.get(), - x_td.get(), - x.implicit(), - y_td.get(), - y.implicit(), - fargs.get()); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("Failed to execute fusion plan"); - return y; - } -}; -#endif - -const std::unordered_set& get_supported_archs() -{ - static std::unordered_set supported_archs{ - "gfx900", "gfx906", "gfx908", "gfx1030", "gfx940"}; - return supported_archs; -} -#if MIGRAPHX_USE_MIOPEN -MIGRAPHX_PRED_MATCHER(bias_shape, instruction_ref ins) -{ - auto&& s = ins->get_shape(); - return s.broadcasted() and s.strides().size() == 4 and s.strides()[0] == 0 and - s.strides()[1] != 0 and s.strides()[2] == 0 and s.strides()[3] == 0; -} - -MIGRAPHX_PRED_MATCHER(fusable_conv, instruction_ref ins) -{ - const auto device_name = trim(split_string(get_device_name(), ':').front()); - if(not contains(get_supported_archs(), device_name)) - return false; - if(enabled(MIGRAPHX_DISABLE_MIOPEN_FUSION{})) - return false; - if(ins->name() != "gpu::convolution") - return false; - if(ins->get_shape().type() != shape::float_type) - return false; - auto wei = ins->inputs().at(1)->get_shape(); - assert(wei.lens().size() == 4); - auto miopen_conv_op = ins->get_operator().to_value(); - auto algo = miopen_conv_op.at("algo").to(); - auto conv_op = from_value(miopen_conv_op["op"]); - if(conv_op.group > 1) - return false; - if(wei.lens()[1] > 512 and algo != miopenConvolutionFwdAlgoWinograd) - return false; - - // Do not fuse non-symmetric input - auto input_lens = ins->inputs().at(0)->get_shape().lens(); - if(input_lens[2] != input_lens[3] or wei.lens()[2] != wei.lens()[3]) - return false; - - // Dont fuse winograd for non-3x3s since there is no fused windograd for those configs - if(algo == miopenConvolutionFwdAlgoWinograd and wei.lens()[2] != 3 and wei.lens()[3] != 3 and - contains({{1, 1}}, conv_op.stride)) - return false; - return contains({{0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}}, conv_op.padding) and - contains({{0, 0}, {1, 1}}, conv_op.stride) and contains({{1, 1}}, conv_op.dilation); -} -#endif - -void move_broadcasted_back(std::vector& args) -{ - // Ensure the last arguments is the broadcasted one - auto last = std::prev(args.end()); - auto it = - std::find_if(args.begin(), last, [](auto arg) { return arg->get_shape().broadcasted(); }); - if(it != last) - std::swap(*it, *std::prev(last)); -} - -void move_standard_front(std::vector& args) -{ - // Ensure the first arguments is the standard one - auto last = std::prev(args.end()); - auto it = - std::find_if(args.begin(), last, [](auto arg) { return arg->get_shape().standard(); }); - if(it != last) - std::swap(*it, args.front()); -} - -auto gpu_name(const std::string& s) { return match::name("gpu::" + s); } - -namespace { -#if MIGRAPHX_USE_MIOPEN -struct miopen_fusion -{ - struct fuse_op_data - { - operation op; - float alpha = 1; - float beta = 0; - }; - struct fuse_op : fuse_op_data, reflect_equality, reflect_stream - { - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op"), f(self.alpha, "alpha"), f(self.beta, "beta")); - } - }; - std::vector ops = {}; - fusion f = {}; - std::function&)> execute; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.ops, "ops")); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } - - value compile(context& ctx, const shape&, std::vector inputs) - { - // Compensate for allocation - inputs.pop_back(); - std::size_t i = 0; - f = fusion(inputs[i]); - i++; - std::vector&)>> - invokers; - for(auto&& fop : ops) - { - if(i > inputs.size()) - { - f = {}; - return {}; - } - if(fop.op.name() == "convolution") - { - auto* mop = f.create_conv(any_cast(fop.op), inputs[i]); - invokers.push_back( - [=](const fused_operator_args& fargs, const std::vector& args) { - miopenSetOpArgsConvForward( - fargs.get(), mop, &fop.alpha, &fop.beta, args[i].implicit()); - }); - i++; - } - else if(fop.op.name() == "add") - { - auto* mop = f.create_bias(inputs[i]); - invokers.push_back( - [=](const fused_operator_args& fargs, const std::vector& args) { - miopenSetOpArgsBiasForward( - fargs.get(), mop, &fop.alpha, &fop.beta, args[i].implicit()); - }); - i++; - } - else if(fop.op.name() == "relu") - { - auto* mop = f.create_relu(); - invokers.push_back([=](const fused_operator_args& fargs, - const std::vector&) { - miopenSetOpArgsActivForward(fargs.get(), mop, &fop.alpha, &fop.beta, 0, 0, 0); - }); - } - else - { - f = {}; - return {}; - } - } - if(not f.compile(ctx)) - { - f = {}; - return {}; - } - execute = [invokers](context& c, const fusion& ff, const std::vector& args) { - auto fargs = make_fused_args(); - for(auto&& invoker : invokers) - invoker(fargs, args); - ff.execute(c, fargs, args.front(), args.back()); - }; - return {{"workspace", f.get_workspace(ctx).bytes()}}; - } - void finalize(context& ctx, const shape& output_shape, const std::vector& inputs) - { - if(not f.empty()) - return; - auto v = compile(ctx, output_shape, inputs); - if(not v.is_object()) - MIGRAPHX_THROW("Failed to compile fusion plan"); - } - std::string name() const { return "gpu::miopen_fusion"; } - shape compute_shape(const std::vector& inputs) const - { - if(ops.empty()) - return {}; - // TODO: Check number of arguments - return ops.front().op.compute_shape({inputs[0], inputs[1]}); - } - argument compute(context& ctx, const shape&, const std::vector& args) const - { - execute(ctx, f, args); - return args.back(); - } -}; -MIGRAPHX_REGISTER_OP(miopen_fusion) - -struct miopen_conv_bias -{ - op::convolution op; - fusion fp = {}; - fusion::op_t conv = {}; - fusion::op_t bias = {}; - - template - static auto reflect(Self& self, F f) - { - return op::convolution::reflect(self.op, f); - } - - std::string name() const { return "gpu::conv_bias"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(5); - // TODO: Check slices - return op.normalize_compute_shape({inputs.at(0), inputs.at(1)}); - } - argument compute(context& ctx, const shape&, const std::vector& args) const - { - auto fargs = make_fused_args(); - float alpha = 1; - float beta = 0; - miopenSetOpArgsConvForward(fargs.get(), conv, &alpha, &beta, args[1].implicit()); - miopenSetOpArgsBiasForward(fargs.get(), bias, &alpha, &beta, args[3].implicit()); - return fp.execute(ctx, fargs, args[0], args[4]); - } - - void finalize(context& ctx, const shape&, const std::vector& inputs) - { - fp = fusion(inputs[0]); - conv = fp.create_conv(op, inputs[1]); - bias = fp.create_bias(inputs[3]); - if(not fp.compile(ctx)) - MIGRAPHX_THROW("Failed to compile fusion plan"); - } - - shape get_workspace(context& ctx) { return fp.get_workspace(ctx); } - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -MIGRAPHX_REGISTER_OP(miopen_conv_bias) - -struct miopen_conv_bias_relu -{ - op::convolution op; - fusion fp = {}; - fusion::op_t conv = {}; - fusion::op_t bias = {}; - fusion::op_t relu = {}; - - template - static auto reflect(Self& self, F f) - { - return op::convolution::reflect(self.op, f); - } - - std::string name() const { return "gpu::conv_bias_relu"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(5); - // TODO: Check slices - return op.normalize_compute_shape({inputs.at(0), inputs.at(1)}); - } - argument compute(context& ctx, const shape&, const std::vector& args) const - { - auto fargs = make_fused_args(); - float alpha = 1; - float beta = 0; - miopenSetOpArgsConvForward(fargs.get(), conv, &alpha, &beta, args[1].implicit()); - miopenSetOpArgsBiasForward(fargs.get(), bias, &alpha, &beta, args[3].implicit()); - miopenSetOpArgsActivForward(fargs.get(), relu, &alpha, &beta, 0, 0, 0); - return fp.execute(ctx, fargs, args[0], args[4]); - } - void finalize(context& ctx, const shape&, const std::vector& inputs) - { - fp = fusion(inputs[0]); - conv = fp.create_conv(op, inputs[1]); - bias = fp.create_bias(inputs[3]); - relu = fp.create_relu(); - fp.compile(ctx); - } - - shape get_workspace(context& ctx) { return fp.get_workspace(ctx); } - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -MIGRAPHX_REGISTER_OP(miopen_conv_bias_relu) - -template -auto conv_bias(Ms... ms) -{ - return match::name("gpu::add")( - match::either_arg(0, 1)(bias_shape(match::used_once()).bind("bias"), - fusable_conv(match::used_once()).bind("conv")), - ms...); -} - -template -void apply_conv_bias(context& ctx, module& m, const match::matcher_result& r) -{ - auto conv_ins = r.instructions["conv"]; - auto bias_ins = r.instructions["bias"]; - auto ins = r.result; - auto input_ins = conv_ins->inputs().at(0); - auto weights_ins = conv_ins->inputs().at(1); - auto conv_op = from_value((conv_ins->get_operator()).to_value()["op"]); - auto alloc_ins = ins->inputs().back(); - auto old_ws_ins = conv_ins->inputs().at(2); - - Op cb{conv_op}; - // TODO: Insert ws allocation - auto ws = cb.get_workspace(ctx); - (void)ws; - m.replace_instruction(ins, cb, input_ins, weights_ins, old_ws_ins, bias_ins, alloc_ins); -} -#endif - -template -inline auto precompile_name(Strings... names) // NOLINT -{ - return match::make_basic_pred_matcher([=](instruction_ref ins) { - if(ins->name() != "gpu::precompile_op") - return false; - auto op = from_value(ins->get_operator().to_value().at("op")); - return (contains({names...}, op.name())); - }); -} - -#if MIGRAPHX_USE_MIOPEN -struct find_conv_bias -{ - context* ctx = nullptr; - auto matcher() const - { - auto relu = match::name(std::unordered_set{"gpu::relu"}); - return conv_bias(match::none_of(match::output(relu))); - } - - void apply(module& m, const match::matcher_result& r) const - { - apply_conv_bias(*ctx, m, r); - } -}; - -struct find_conv_bias_relu -{ - context* ctx = nullptr; - auto matcher() const { return match::name("gpu::relu")(match::arg(0)(conv_bias())); } - - void apply(module& m, const match::matcher_result& r) const - { - apply_conv_bias(*ctx, m, r); - } -}; -struct find_conv_pointwise -{ - context* ctx = nullptr; - auto matcher() const - { - return precompile_name("pointwise")( - match::nargs(3), - match::either_arg(0, 1)(bias_shape(match::used_once()).bind("bias"), - fusable_conv(match::used_once()).bind("conv"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto conv_ins = r.instructions["conv"]; - auto bias_ins = r.instructions["bias"]; - auto ins = r.result; - auto input_ins = conv_ins->inputs().at(0); - auto weights_ins = conv_ins->inputs().at(1); - auto conv_op = from_value(conv_ins->get_operator().to_value()["op"]); - auto alloc_ins = ins->inputs().back(); - - module_ref pm = ins->module_inputs().front(); - - miopen_fusion op{}; - op.ops.push_back({{conv_op}}); - for(auto&& i : *pm) - { - if(i.name()[0] == '@') - continue; - op.ops.push_back({{i.get_operator()}}); - } - std::vector inputs = {input_ins, weights_ins, bias_ins, alloc_ins}; - auto v = op.compile(*ctx, ins->get_shape(), to_shapes(inputs)); - if(not v.is_object()) - return; - m.replace_instruction(ins, op, inputs); - } -}; -#endif - -#if MIGRAPHX_USE_ROCBLAS or MIGRAPHX_USE_HIPBLASLT -struct gemm_pointwise -{ - // TODO: Move to matcher.hpp - static auto match_param(const std::string& name) - { - return match::make_basic_pred_matcher([=](auto ins) { - if(ins->name() != "@param") - return false; - auto p = any_cast(ins->get_operator()); - return p.parameter == name; - }); - } - - template - static auto match_mul_const(M m, const std::string& var) - { - return match::name("mul")(match::either_arg(0, 1)(match::name("@literal").bind(var), m)) - .bind(var + "_mul"); - } - - static auto match_add(const std::string& input, const std::string& output) - { - auto param = match::name("@param"); - auto add = match::name("add")(match::args(param, param)); - auto inner_mul = match::any_of(match_mul_const(match_param(input), "alpha"), - match_mul_const(match_param(output), "beta")); - auto mul_add = match::name("add")(match::either_arg(0, 1)(inner_mul, param)); - auto add_mul = match_mul_const(add, "gamma"); - return match::name("@return")(match::args(match::any_of(add, mul_add, add_mul))); - } - - static auto match_mul(const std::string& input) - { - auto mul = match_mul_const(match_param(input), "alpha"); - return match::name("@return")(match::args(mul)); - } - - static float get_float(instruction_ref ins) { return ins->get_literal().at(); } - - template - static bool update_gemm(Gemm& gemm, module_ref pm, unsigned input) - { - auto names = pm->get_parameter_names(); - std::sort(names.begin(), names.end()); - if(names.size() == 1) - { - auto mr = match::match_instruction(*pm, std::prev(pm->end()), match_mul(names[input])); - if(mr.result == pm->end()) - return false; - gemm.alpha *= get_float(mr.instructions["alpha"]); - return true; - } - else if(names.size() == 2) - { - unsigned output = input == 0 ? 1 : 0; - auto mr = match::match_instruction( - *pm, std::prev(pm->end()), match_add(names[input], names[output])); - if(mr.result == pm->end()) - return false; - if(contains(mr.instructions, "alpha_mul")) - gemm.alpha *= get_float(mr.instructions["alpha"]); - else if(contains(mr.instructions, "beta_mul")) - gemm.beta *= get_float(mr.instructions["beta"]); - else if(contains(mr.instructions, "gamma_mul")) - { - gemm.alpha *= get_float(mr.instructions["gamma"]); - gemm.beta *= get_float(mr.instructions["gamma"]); - } - return true; - } - else - { - return false; - } - } -}; -#endif - -#if MIGRAPHX_USE_ROCBLAS -struct find_rocblas_gemm_pointwise : gemm_pointwise -{ - auto matcher() const - { - auto gemm_op = match::name("gpu::gemm")(match::nargs(3), match::used_once()).bind("gemm"); - auto binary_op = match::all_of( - match::nargs(3), - match::either_arg(0, 1)( - match::any_of(match::standard_shape(), match::is_constant()).bind("c"), gemm_op)); - auto unary_op = match::all_of(match::nargs(2), match::arg(0)(gemm_op)); - return precompile_name("pointwise")(match::any_of(binary_op, unary_op)); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto gemm_ins = r.instructions["gemm"]; - - auto gemm = any_cast>(gemm_ins->get_operator()); - - // Already fused gemm - if(not float_equal(gemm.beta, 0)) - return; - if(ins->inputs().size() == 3) - gemm.beta = 1; - - if(not update_gemm( - gemm, ins->module_inputs().front(), ins->inputs().front() == gemm_ins ? 0 : 1)) - return; - - auto inputs = gemm_ins->inputs(); - inputs.pop_back(); - - if(ins->inputs().size() == 3) - { - auto c_ins = r.instructions["c"]; - shape s = c_ins->get_shape(); - // const-fold input if not standard shape since rocblas can't handle it - // Updated for a case where "standard" shape has out-of-sequence strides - if(not s.standard()) - { - auto c = make_op("contiguous"); - auto l = c.compute(c.compute_shape({c_ins->get_shape()}), {c_ins->eval()}); - c_ins = m.add_literal(l.get_shape(), l.data()); - } - inputs.push_back(c_ins); - } - - inputs.push_back(ins->inputs().back()); - - m.replace_instruction(ins, gemm, inputs); - } -}; -#endif - -#if MIGRAPHX_USE_HIPBLASLT -struct find_hipblas_gemm_pointwise : gemm_pointwise -{ - auto matcher() const - { - auto gemm_op = - match::name("gpu::hipblaslt_op")(match::nargs(3), match::used_once()).bind("hip_gemm"); - auto binary_op = match::all_of( - match::nargs(3), - match::either_arg(0, 1)( - match::any_of(match::standard_shape(), match::is_constant()).bind("c"), gemm_op)); - auto unary_op = match::all_of(match::nargs(2), match::arg(0)(gemm_op)); - return precompile_name("pointwise")(match::any_of(binary_op, unary_op)); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto gemm_ins = r.instructions["hip_gemm"]; - - auto gemm_op = any_cast(gemm_ins->get_operator()).op; - - if(gemm_op.name() != "gpu::hip_gemm") - return; - - auto gemm = any_cast>(gemm_op); - - // Already fused gemm - if(not float_equal(gemm.beta, 0)) - return; - if(ins->inputs().size() == 3) - gemm.beta = 1; - if(not update_gemm( - gemm, ins->module_inputs().front(), ins->inputs().front() == gemm_ins ? 0 : 1)) - { - return; - } - auto inputs = gemm_ins->inputs(); - inputs.pop_back(); - if(ins->inputs().size() == 3) - { - auto c_ins = r.instructions["c"]; - shape s = c_ins->get_shape(); - // const-fold input if not standard shape - // Updated for a case where "standard" shape has out-of-sequence strides - if(not s.standard()) - { - auto c = make_op("contiguous"); - auto l = c.compute(c.compute_shape({c_ins->get_shape()}), {c_ins->eval()}); - c_ins = m.add_literal(l.get_shape(), l.data()); - } - inputs.push_back(c_ins); - } - inputs.push_back(ins->inputs().back()); - - operation new_gemm_op = gemm; - auto new_ins = m.insert_instruction( - ins, make_op("gpu::hipblaslt_op", {{"op", to_value(new_gemm_op)}}), inputs); - m.replace_instruction(ins, new_ins); - } -}; -#endif - -struct contiguous_transpose_gemm -{ - template - static bool is_swapped(const Vector& perm, std::size_t i, std::size_t j) - { - if(i >= perm.size() or j >= perm.size()) - return false; - auto perm2 = perm; - std::iota(perm2.begin(), perm2.end(), 0); - std::swap(perm2[i], perm2[j]); - return perm2 == perm; - } -}; - -struct find_contiguous_transpose_rocblas_gemm : contiguous_transpose_gemm -{ - auto matcher() const - { - return match::name("gpu::contiguous")(match::arg(0)( - match::name("transpose")( - match::arg(0)(match::name("gpu::gemm")(match::used_once()).bind("gemm"))) - .bind("transpose"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto gemm = r.instructions["gemm"]; - auto alloc = gemm->inputs().back(); - auto transpose = r.instructions["transpose"]; - auto perm = transpose->get_operator().to_value()["permutation"].to_vector(); - auto iperm = invert_permutation(perm); - - if(perm.size() < 3) - return; - - if(not is_swapped(perm, perm.size() - 3, perm.size() - 2)) - return; - - auto lens = gemm->get_shape().lens(); - if(lens.size() > 3 and - not std::all_of(lens.begin(), lens.end() - 3, [](auto i) { return i == 1; })) - return; - - auto gemmv = gemm->get_operator().to_value(); - gemmv["trans_batch"] = 1; - - auto s = shape{alloc->get_shape().type(), reorder_dims(alloc->get_shape().lens(), iperm)}; - auto new_alloc = m.insert_instruction(gemm, make_op("allocate", {{"shape", to_value(s)}})); - auto alloc_transpose = - m.insert_instruction(gemm, make_op("transpose", {{"permutation", perm}}), new_alloc); - - auto inputs = gemm->inputs(); - inputs.back() = alloc_transpose; - auto new_gemm = m.insert_instruction(gemm, make_op("gpu::gemm", gemmv), inputs); - auto gemm_transpoe = m.insert_instruction(gemm, transpose->get_operator(), new_gemm); - - m.replace_instruction(ins, gemm_transpoe); - } -}; - -#if MIGRAPHX_USE_HIPBLASLT -struct find_contiguous_transpose_hip_gemm : contiguous_transpose_gemm -{ - auto matcher() const - { - return match::name("gpu::contiguous")(match::arg(0)( - match::name("transpose")( - match::arg(0)( - match::name("gpu::hipblaslt_op")(match::used_once()).bind("hip_gemm"))) - .bind("transpose"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto gemm_ins = r.instructions["hip_gemm"]; - auto gemm_op = any_cast(gemm_ins->get_operator()).op; - - if(gemm_op.name() != "gpu::hip_gemm") - return; - - auto gemm = any_cast>(gemm_op); - - auto alloc = gemm_ins->inputs().back(); - auto transpose = r.instructions["transpose"]; - auto perm = transpose->get_operator().to_value()["permutation"].to_vector(); - auto iperm = invert_permutation(perm); - - if(perm.size() < 3) - return; - - if(not is_swapped(perm, perm.size() - 3, perm.size() - 2)) - return; - - auto lens = gemm_ins->get_shape().lens(); - if(lens.size() > 3 and - not std::all_of(lens.begin(), lens.end() - 3, [](auto i) { return i == 1; })) - return; - - gemm.trans_batch = 1; - - auto s = shape{alloc->get_shape().type(), reorder_dims(alloc->get_shape().lens(), iperm)}; - auto new_alloc = - m.insert_instruction(gemm_ins, make_op("allocate", {{"shape", to_value(s)}})); - - auto alloc_transpose = m.insert_instruction( - gemm_ins, make_op("transpose", {{"permutation", perm}}), new_alloc); - - auto inputs = gemm_ins->inputs(); - inputs.back() = alloc_transpose; - operation new_gemm_op = gemm; - auto new_gemm = m.insert_instruction( - gemm_ins, make_op("gpu::hipblaslt_op", {{"op", to_value(new_gemm_op)}}), inputs); - - auto gemm_transpoe = m.insert_instruction(gemm_ins, transpose->get_operator(), new_gemm); - - m.replace_instruction(ins, gemm_transpoe); - } -}; -#endif - -struct find_commutative_broadcast -{ - auto matcher() const - { - return match::name("gpu::add", "gpu::mul")(match::arg(1)(match::broadcast_shape())); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto args = ins->inputs(); - move_broadcasted_back(args); - - m.replace_instruction(ins, ins->get_operator(), args); - } -}; -} // namespace - -struct find_contiguous -{ - auto matcher() const { return match::name("gpu::contiguous"); } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - - m.replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(make_op("contiguous"))}}), - ins->inputs()); - } -}; - -struct find_contiguous_layout_pointwise -{ - auto matcher() const - { - auto cont_pw = precompile_name("pointwise")(match::any_of[match::inputs()]( - match::name("gpu::contiguous")(match::used_once()).bind("layout_ins"))); - auto layout_pw = precompile_name("pointwise")(match::any_of[match::inputs()]( - precompile_name("layout")(match::used_once()).bind("layout_ins"))); - return match::any_of(cont_pw, layout_pw); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto pw_ins = r.result; - auto layout_ins = r.instructions["layout_ins"]; - auto layout_input = layout_ins->inputs().front(); - auto pw_ins_inputs = pw_ins->inputs(); - replace(pw_ins_inputs, layout_ins, layout_input); - // Ensure the output shape of the pointwise module retains the memory layout - auto pw_op_val = pw_ins->get_operator().to_value(); - pw_op_val["output_shape"] = to_value(pw_ins->get_shape()); - - auto new_ins = m.insert_instruction( - pw_ins, make_op(pw_ins->name(), pw_op_val), pw_ins_inputs, pw_ins->module_inputs()); - m.replace_instruction(pw_ins, new_ins); - } -}; - -struct find_pointwise_layout_contiguous -{ - auto matcher() const - { - auto is_layout = precompile_name("layout")( - match::arg(0)(match::used_once(), precompile_name("pointwise"))); - auto is_contiguous = match::name("gpu::contiguous")( - match::arg(0)(match::used_once(), precompile_name("pointwise"))); - return match::any_of(is_layout, is_contiguous); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto pw = ins->inputs().front(); - auto alloc = ins->inputs().back(); - auto args = pw->inputs(); - args.back() = alloc; - - // Ensure the output shape of the pointwise module retains the memory layout - auto pw_op_val = pw->get_operator().to_value(); - pw_op_val["output_shape"] = to_value(ins->get_shape()); - - m.replace_instruction(ins, make_op(pw->name(), pw_op_val), args, pw->module_inputs()); - } -}; - -struct find_layernorm_pointwise -{ - auto matcher() const - { - return precompile_name("pointwise")(match::arg(0)( - precompile_name("gpu::prelayernorm", "gpu::preadd_layernorm").bind("layernorm"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto pw_ins = r.result; - auto layernorm = r.instructions["layernorm"]; - if(not layernorm->module_inputs().empty()) - return; - auto* pm = pw_ins->module_inputs().front(); - auto pw_inputs = pw_ins->inputs(); - auto ln_pos = std::find(pw_inputs.begin(), pw_inputs.end(), layernorm); - assert(ln_pos != pw_inputs.end()); - pw_inputs.erase(ln_pos); - auto inputs = layernorm->inputs(); - inputs.pop_back(); - inputs.insert(inputs.end(), pw_inputs.begin(), pw_inputs.end()); - - // Ensure the output shape retains the memory layout - auto layernorm_op_val = layernorm->get_operator().to_value(); - layernorm_op_val["output_shape"] = to_value(pw_ins->get_shape()); - - m.replace_instruction(pw_ins, make_op(layernorm->name(), layernorm_op_val), inputs, {pm}); - } -}; - -struct find_concat_pointwise -{ - auto matcher() const - { - return precompile_name("pointwise")( - match::arg(0)(precompile_name("concat").bind("concat"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto concat = r.instructions["concat"]; - if(not concat->module_inputs().empty()) - return; - - // TODO: Handle type conversions - if(ins->get_shape().type() != concat->get_shape().type()) - return; - - auto* pm = ins->module_inputs().front(); - auto inputs = concat->inputs(); - inputs.pop_back(); - inputs.insert(inputs.end(), ins->inputs().begin() + 1, ins->inputs().end()); - - auto op = concat->get_operator(); - op.from_value({{"additional_args", ins->inputs().size() - 1}, - {"ignore_modules", true}, - {"output_shape", to_value(ins->get_shape())}}); - - m.replace_instruction(ins, op, inputs, {pm}); - } -}; - -void fuse_ops::apply(module& m) const -{ - match::find_matches(m, find_pointwise_layout_contiguous{}, find_contiguous_layout_pointwise{}); - run_passes(m, {dead_code_elimination{}}); -#if MIGRAPHX_USE_MIOPEN - match::find_matches(m, find_conv_pointwise{ctx}, find_conv_bias_relu{ctx}, find_conv_bias{ctx}); - run_passes(m, {dead_code_elimination{}}); -#endif -#if MIGRAPHX_USE_ROCBLAS - match::find_matches(m, find_rocblas_gemm_pointwise{}); -#endif -#if MIGRAPHX_USE_HIPBLASLT - match::find_matches(m, find_hipblas_gemm_pointwise{}); -#endif - match::find_matches(m, - find_layernorm_pointwise{}, - find_concat_pointwise{}, - find_contiguous_transpose_rocblas_gemm{}, -#if MIGRAPHX_USE_HIPBLASLT - find_contiguous_transpose_hip_gemm{}, -#endif - find_commutative_broadcast{}); - match::find_matches(m, find_contiguous{}); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/gemm_impl.cpp b/docker/rocm/migraphx/targets/gpu/gemm_impl.cpp deleted file mode 100644 index d0f750a25..000000000 --- a/docker/rocm/migraphx/targets/gpu/gemm_impl.cpp +++ /dev/null @@ -1,708 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -using microseconds = std::chrono::duration; - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_ROCBLAS -/* -Regular rocBLAS API takes compute_type as `rocblas_datatype` enum value v/s "ex3" BETA API takes it -as `rocblas_computetype` enum value. `rb_compute_type` is faciliator to implictly cast integer enum -value to required type that can be used inside `common_args` generator. -*/ -struct rb_compute_type -{ - int type = 0; - rb_compute_type(rocblas_datatype t) : type(static_cast(t)) {} - rb_compute_type(rocblas_computetype t) : type(static_cast(t)) {} - operator rocblas_datatype() const { return static_cast(type); } - operator rocblas_computetype() const { return static_cast(type); } -}; - -// Convert rocBLAS datatypes to equivalent Migraphx data types -rocblas_datatype get_type(shape::type_t type) -{ - switch(type) - { - case shape::double_type: return rocblas_datatype_f64_r; - case shape::float_type: return rocblas_datatype_f32_r; - case shape::half_type: return rocblas_datatype_f16_r; - case shape::int8_type: return rocblas_datatype_i8_r; - case shape::uint8_type: return rocblas_datatype_u8_r; - case shape::int32_type: return rocblas_datatype_i32_r; - case shape::uint32_type: return rocblas_datatype_u32_r; - case shape::fp8e4m3fnuz_type: return rocblas_datatype_f8_r; - case shape::fp8e5m2fnuz_type: return rocblas_datatype_bf8_r; - case shape::fp8e4m3fn_type: - case shape::fp8e5m2_type: - case shape::tuple_type: - case shape::bool_type: - case shape::uint16_type: - case shape::int16_type: - case shape::int64_type: - case shape::uint64_type: MIGRAPHX_THROW("ROCBLAS_GEMM: data type not supported!"); - case shape::bf16_type: return rocblas_datatype_bf16_r; - } - - MIGRAPHX_THROW("ROCBLAS_GEMM: data type not supported!"); -} - -void blas_shape(const shape& in_shape) -{ - if(in_shape.lens().size() < 2) - return; - auto s = in_shape.normalize_standard(); - if(std::none_of(s.strides().end() - 2, s.strides().end(), [](auto i) { return i == 1; })) - MIGRAPHX_THROW("GPU_GEMM: needs to have one matrix stride as 1"); - if(std::any_of(s.strides().end() - 2, s.strides().end(), [](auto i) { return i == 0; })) - MIGRAPHX_THROW("GPU_GEMM: matrix dimensions can't be broadcasted"); - if(s.lens().size() < 3) - return; - shape batch_shape{s.type(), - {s.lens().begin(), s.lens().end() - 2}, - {s.strides().begin(), s.strides().end() - 2}}; - auto batch_shapes = reduce_dims({batch_shape}); - if(batch_shapes.front().lens().size() != 1) - MIGRAPHX_THROW("GPU_GEMM: Batch dimension is not collapsible"); -} - -shape transpose_batch(const shape& s, unsigned trans_batch) -{ - if(trans_batch == 0) - return s; - if(s.lens().size() < 3) - return s; - auto batch = s.lens().size() - 3; - std::vector perm(s.lens().size()); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[batch], perm[batch + trans_batch]); - return shape::from_permutation(s.type(), s.lens(), perm); -} - -/** - * Returns results of rocblas_status_success, rocblas_status_perf_degraded, - * or rocblas_status_invalid_value. Caller - * is expected to check for invalid index. Any other result causes an exception. - * - */ -template -auto rocblas_invoke(F f, Pack p, Ts... xs) -{ - return p([=](auto... ws) { - auto status = f(ws..., xs...); - if(status != rocblas_status_success and status != rocblas_status_invalid_value) - { - if(status == rocblas_status_perf_degraded) - { - std::cerr << "WARNING: degraded perf. in rocBLAS call" << std::endl; - } - else - MIGRAPHX_THROW("rocblas_invoke: rocBLAS call failed with status " + - std::to_string(status)); - } - return status; - }); -} - -static bool is_transposed(const shape& s) -{ - if(s.transposed()) - { - return s.strides().back() != 1; - } - - if(not s.broadcasted() and s.strides() != s.as_standard().strides()) - { - auto perm = find_permutation(s); - return not std::is_sorted(perm.begin(), perm.end()); - } - - return false; -} - -static rocblas_int get_batch_stride(const shape& s) -{ - // This value is not needed for non-strided inputs - if(s.strides().size() < 3) - return 0; - else - return s.strides()[s.strides().size() - 3]; -} - -/** - * Wrapper for multiple rocBLAS calls. The constructor creates parameters for - * these calls based on data shapes and other values contained in the associated - * instruction and operation. - * - * The template parameter T is not the type of the matrix data but of the weighting - * coefficients alpha and beta (these are float in rocBLAS internals) - */ -template -struct gemm_impl -{ - gemm_impl(const shape& output_shape, - const std::vector& input_shapes, - T alpha_param, - T beta_param, - bool compute_fp32_flag) - : alpha(alpha_param), - beta(beta_param), - is_3inputs(input_shapes.size() == 4), - compute_fp32(compute_fp32_flag) - { - if(not is_3inputs) - { - beta = 0; - } - - // Create lambdas that will cast alpha, beta to the output shape's type - // and retain the values being pointed to - output_shape.visit_type([&](auto as) { - auto alpha_r = as(alpha); - auto beta_r = as(beta); - if(compute_fp32) - { - get_alpha = [=] { return α }; - get_beta = [=] { return β }; - } - else - { - get_alpha = [=] { return &alpha_r; }; - get_beta = [=] { return &beta_r; }; - } - }); - - transa = is_transposed(input_shapes[0]); - transb = is_transposed(input_shapes[1]); - auto n_dim = output_shape.lens().size(); - auto dim_0 = n_dim - 2; - auto dim_1 = n_dim - 1; - // Leading dimensions of matrices - lda = input_shapes[0].strides()[transa ? dim_1 : dim_0]; - ldb = input_shapes[1].strides()[transb ? dim_1 : dim_0]; - ldc = input_shapes[2].strides()[dim_0]; - ldd = is_3inputs ? input_shapes[3].strides()[dim_0] : ldc; - - arg_type = get_type(input_shapes[0].type()); - output_type = get_type(input_shapes[2].type()); - if(output_type == rocblas_datatype_i8_r or output_type == rocblas_datatype_u8_r) - { - output_type = rocblas_datatype_i32_r; - } - compute_type = rb_compute_type{output_type}; - if(compute_fp32) - { - if(arg_type == rocblas_datatype_f16_r or arg_type == rocblas_datatype_bf16_r) - compute_type = rocblas_datatype_f32_r; - } - if(arg_type == rocblas_datatype_f8_r) - { - assert(get_type(input_shapes[1].type()) == rocblas_datatype_f8_r); - compute_type = rocblas_compute_type_f32; - } - - auto a_lens = input_shapes[0].lens(); - auto b_lens = input_shapes[1].lens(); - - auto out_lens = output_shape.lens(); - m = out_lens[dim_0]; - n = out_lens[dim_1]; - k = input_shapes[0].lens()[dim_1]; - - a_stride = get_batch_stride(input_shapes[0]); - b_stride = get_batch_stride(input_shapes[1]); - c_stride = get_batch_stride(input_shapes[2]); - d_stride = is_3inputs ? get_batch_stride(input_shapes[3]) : c_stride; - num_matrices = std::accumulate( - out_lens.rbegin() + 2, out_lens.rend(), std::size_t{1}, std::multiplies()); - strided_batched = num_matrices > 1; - if(strided_batched and b_stride == 0 and input_shapes[0].standard()) - { - // If the batch dimension of B is broadcasted, then we can - // multiply m by the batch_size and use rocblas_gemm_ex - // instead of rocblas_gemm_strided_batched_ex. - m *= num_matrices; - strided_batched = false; - } - } - - void run(context& ctx, const std::vector& input_args, int32_t solution_idx = 0) const - { -#ifdef MIGRAPHX_USE_ROCBLAS_FP8_API - if(rocblas_fp8_available() and - std::any_of(input_args.begin(), input_args.end(), [](const auto i) { - return i.get_shape().type() == migraphx::shape::fp8e4m3fnuz_type; - })) - { - if(strided_batched) - { - auto common_args = - create_strided_batched_args_common(ctx, compute_type, input_args); - rocblas_invoke(&rocblas_gemm_strided_batched_ex3, - common_args, - rocblas_gemm_algo_standard, - solution_idx, - gemm_flags); - } - else - { - auto common_args = create_gemm_ex_args_common(ctx, compute_type, input_args); - rocblas_invoke(&rocblas_gemm_ex3, - common_args, - rocblas_gemm_algo_standard, - solution_idx, - gemm_flags); - } - } - else -#endif - { - if(strided_batched) - { - auto common_args = - create_strided_batched_args_common(ctx, compute_type, input_args); - rocblas_invoke(&rocblas_gemm_strided_batched_ex, - common_args, - rocblas_gemm_algo_solution_index, - solution_idx, - gemm_flags); - } - else - { - auto common_args = create_gemm_ex_args_common(ctx, compute_type, input_args); - rocblas_invoke(&rocblas_gemm_ex, - common_args, - rocblas_gemm_algo_solution_index, - solution_idx, - gemm_flags); - } - } - } - -#ifdef MIGRAPHX_USE_ROCBLAS_TUNING_API - auto validate(context& ctx, const std::vector& input_shapes, int32_t solution_idx) const - { - // Create dummy arguments for the shapes, and call the overloaded method - std::vector input_args; - unsigned long seed = 0; - std::transform(input_shapes.begin(), - input_shapes.end(), - std::back_inserter(input_args), - [&](const shape& x) { - return to_gpu(generate_argument(x, seed++, random_mode::random)); - }); - return validate(ctx, input_args, solution_idx); - } - - /** - * Checks a particular solution for validity by running it with the flag - * rocblas_gemm_flags_check_solution_index (could be invalid if this model was - * tuned with a different rocBLAS version) - * - * @return Returns either solution_idx if valid, or else the default value 0 - * if not. The default does not mean list index 0, but tells the picker - * to choose a solution. - */ - int32_t - validate(context& ctx, const std::vector& input_args, int32_t solution_idx) const - { - rocblas_status_ check_valid(rocblas_status_success); - - if(strided_batched) - { - auto common_args = create_strided_batched_args_common(ctx, compute_type, input_args); - check_valid = rocblas_invoke(&rocblas_gemm_strided_batched_ex, - common_args, - rocblas_gemm_algo_solution_index, - solution_idx, - rocblas_gemm_flags_check_solution_index); - } - else - { - auto common_args = create_gemm_ex_args_common(ctx, compute_type, input_args); - check_valid = rocblas_invoke(&rocblas_gemm_ex, - common_args, - rocblas_gemm_algo_solution_index, - solution_idx, - rocblas_gemm_flags_check_solution_index); - } - - if(check_valid == rocblas_status_invalid_value) - { - std::cerr << "WARNING: tuned solution is invalid; reverting to default" << std::endl; - return 0; - } - return solution_idx; - } -#endif - - /** - * Helper method to create that subset of a long rocBLAS argument list that is common - * to multiple "...strided_batched..." calls. - * - * The rocblas_gemm API handles inputs and output matrices as - * column-major format. When doing a C = A * B, we actually do - * C^T = (B^T) * (A^T). That is the reason we input args[1] as - * A and args[0] as B in calling the rocblas_gemm. - * - */ - auto create_strided_batched_args_common(context& ctx, - rb_compute_type rbcompute_type, - const std::vector& args) const - { - return pack(ctx.get_stream().get_rocblas(), - transb ? rocblas_operation_transpose : rocblas_operation_none, - transa ? rocblas_operation_transpose : rocblas_operation_none, - n, - m, - k, - get_alpha(), - args[1].data(), - arg_type, - ldb, - b_stride, - args[0].data(), - arg_type, - lda, - a_stride, - get_beta(), - args[2].data(), - output_type, - ldc, - c_stride, - is_3inputs ? args[3].data() : args[2].data(), - output_type, - ldd, - d_stride, - num_matrices, - rbcompute_type); - } - /** - * Helper method to create that subset of a long rocBLAS argument list that is common - * to multiple "gemm_ex..." calls. - * - * The rocblas_gemm API handles inputs and output matrices as - * column-major format. When doing a C = A * B, we actually do - * C^T = (B^T) * (A^T). That is the reason we input args[1] as - * A and args[0] as B in calling the rocblas_gemm. - * - * */ - auto create_gemm_ex_args_common(context& ctx, - rb_compute_type rbcompute_type, - const std::vector& args) const - { - return pack(ctx.get_stream().get_rocblas(), - transb ? rocblas_operation_transpose : rocblas_operation_none, - transa ? rocblas_operation_transpose : rocblas_operation_none, - n, - m, - k, - get_alpha(), - args[1].data(), - arg_type, - ldb, - args[0].data(), - arg_type, - lda, - get_beta(), - args[2].data(), - output_type, - ldc, - is_3inputs ? args[3].data() : args[2].data(), - output_type, - ldd, - rbcompute_type); - } - -#ifdef MIGRAPHX_USE_ROCBLAS_TUNING_API - /** - * Find best rocBLAS solution: Get list of solutions and try them all, returning the index - * of the fastest one. - */ - int tune(context& ctx, const std::vector& input_shapes) const - { - // tuning meta parameters - const int hot_calls = 40; - unsigned long seed = 0; - std::vector input_args; - std::transform(input_shapes.begin(), - input_shapes.end(), - std::back_inserter(input_args), - [&](const shape& x) { - return to_gpu(generate_argument(x, seed++, random_mode::random)); - }); - - // Get the solutions list in 2 rocBLAS steps: - // 1. Find out how many solutions there are and allocate the array - // 2. Get the solutions - // - rocblas_int list_size = 0; - std::vector solution_indices; - rb_compute_type rbcompute_type = compute_type; - // rocblas_gemm_get_solutions() API requires compute_type as rocblas_datatype. Convert - // manually for FP8 - if(arg_type == rocblas_datatype_f8_r) - { - rbcompute_type = rocblas_datatype_f32_r; - } - if(strided_batched) - { - auto common_args = create_strided_batched_args_common(ctx, rbcompute_type, input_args); - rocblas_invoke(&rocblas_gemm_strided_batched_ex_get_solutions, - common_args, - rocblas_gemm_algo_solution_index, - gemm_flags, - nullptr, - &list_size); - solution_indices.resize(list_size); - - auto common_sol_args = - create_strided_batched_args_common(ctx, rbcompute_type, input_args); - rocblas_invoke(&rocblas_gemm_strided_batched_ex_get_solutions, - common_sol_args, - rocblas_gemm_algo_solution_index, - gemm_flags, - solution_indices.data(), - &list_size); - } - else - { - auto common_args = create_gemm_ex_args_common(ctx, rbcompute_type, input_args); - rocblas_invoke(&rocblas_gemm_ex_get_solutions, - common_args, - rocblas_gemm_algo_solution_index, - gemm_flags, - nullptr, - &list_size); - solution_indices.resize(list_size); - - auto common_sol_args = create_gemm_ex_args_common(ctx, rbcompute_type, input_args); - rocblas_invoke(&rocblas_gemm_ex_get_solutions, - common_sol_args, - rocblas_gemm_algo_solution_index, - gemm_flags, - solution_indices.data(), - &list_size); - } - - double best_time = std::numeric_limits::max(); - double first_time = -1; - // Initialize to default solution index - rocblas_int best_sol = 0; - for(auto sol : solution_indices) - { - // Warmup: the first call to an op. may not be representative since there is - // more time taken initializing caches, etc. so we won't time it. - run(ctx, input_args, sol); - double host_time = time([&] { - for([[maybe_unused]] int hc : range(hot_calls)) - run(ctx, input_args, sol); - ctx.finish(); - }); - - host_time /= hot_calls; - - // dev/evaluation only: track time for first solution. - if(first_time < 0) - first_time = host_time; - - // track current best - if(host_time < best_time) - { - best_sol = sol; - best_time = host_time; - } - } - std::cout << "Winning GEMM solution: " << best_sol << " in " << best_time << " ms, beats " - << first_time << "ms" << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds{50}); - return best_sol; - } -#endif - private: - size_t num_matrices = 0; - rocblas_int m = 0; - rocblas_int n = 0; - rocblas_int k = 0; - bool transa = false; - bool transb = false; - T alpha = 0; - T beta = 0; - - std::function get_alpha{}; - std::function get_beta{}; - rocblas_gemm_flags gemm_flags = rocblas_gemm_flags_none; - rocblas_int lda = 0; - rocblas_int ldb = 0; - rocblas_int ldc = 0; - rocblas_int ldd = 0; - rocblas_int a_stride = 0; - rocblas_int b_stride = 0; - rocblas_int c_stride = 0; - rocblas_int d_stride = 0; - rocblas_datatype arg_type = rocblas_datatype_f32_r; - rb_compute_type compute_type = rocblas_datatype_f32_r; - rocblas_datatype output_type = rocblas_datatype_f32_r; - bool strided_batched = true; - bool is_3inputs = true; - bool compute_fp32 = true; -}; // gemm_impl - -void gemm_compute(context& ctx, - const shape& output_shape, - const std::vector& args, - float alpha, - float beta, - bool compute_fp32, - int32_t solution_idx) -{ - std::vector input_shapes; - std::transform(args.begin(), - args.end(), - std::back_inserter(input_shapes), - [](const argument& x) { return x.get_shape().normalize_standard(); }); - auto gemm_item = gemm_impl(output_shape, input_shapes, alpha, beta, compute_fp32); - gemm_item.run(ctx, args, solution_idx); -} - -void gemm_compute(context& ctx, - const shape& output_shape, - const std::vector& args, - int32_t alpha, - int32_t beta, - bool compute_fp32, - int32_t solution_idx) -{ - std::vector input_shapes; - std::transform(args.begin(), - args.end(), - std::back_inserter(input_shapes), - [](const argument& x) { return x.get_shape().normalize_standard(); }); - auto gemm_item = gemm_impl(output_shape, input_shapes, alpha, beta, compute_fp32); - gemm_item.run(ctx, args, solution_idx); -} - -static value gemm_problem(const shape& output_shape, std::vector input_shapes) -{ - input_shapes.push_back(output_shape); - return to_value(input_shapes); -} - -#ifdef MIGRAPHX_USE_ROCBLAS_TUNING_API -static void gemm_save_solution(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - int32_t solution_idx) -{ - ctx.get_problem_cache().insert( - "rocblas", gemm_problem(output_shape, input_shapes), solution_idx); -} -#endif - -int32_t gemm_default_solution(context& ctx, - const shape& output_shape, - const std::vector& input_shapes) -{ - auto sol = ctx.get_problem_cache().get("rocblas", gemm_problem(output_shape, input_shapes)); - if(sol.has_value()) - return sol->to(); - return 0; -} - -/** - * Decides if the tune() or validate() method is appropriate and calls it. - * Return value is the chosen solution index, or 0 to let picker choose it. - */ -template -int32_t gemm_finalize_impl(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - T alpha, - T beta, - bool compute_fp32, - int32_t solution_idx) -{ -#ifdef MIGRAPHX_USE_ROCBLAS_TUNING_API - - // This code should be called only if either the environment var. - // MIGRAPHX_ENABLE_GEMM_TUNING, or option --exhaustive-tune, is set - - if(solution_idx == 0) - { - auto gemm_item = gemm_impl(output_shape, input_shapes, alpha, beta, compute_fp32); - solution_idx = gemm_item.tune(ctx, input_shapes); - gemm_save_solution(ctx, output_shape, input_shapes, solution_idx); - } - else - { - // If a tuned solution index is already given, don't tune again but validate - // in case the data was tuned with a different rocBLAS version - auto gemm_item = gemm_impl(output_shape, input_shapes, alpha, beta, compute_fp32); - solution_idx = gemm_item.validate(ctx, input_shapes, solution_idx); - } -#else - (void)ctx, (void)output_shape, (void)input_shapes; - (void)alpha, (void)beta, (void)compute_fp32; -#endif - return solution_idx; -} - -int32_t gemm_finalize(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - float alpha, - float beta, - bool compute_fp32, - int32_t solution_idx) -{ - return gemm_finalize_impl( - ctx, output_shape, input_shapes, alpha, beta, compute_fp32, solution_idx); -} - -int32_t gemm_finalize(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - int32_t alpha, - int32_t beta, - bool compute_fp32, - int32_t solution_idx) -{ - return gemm_finalize_impl( - ctx, output_shape, input_shapes, alpha, beta, compute_fp32, solution_idx); -} -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/hip.cpp b/docker/rocm/migraphx/targets/gpu/hip.cpp deleted file mode 100644 index 0fb7deb93..000000000 --- a/docker/rocm/migraphx/targets/gpu/hip.cpp +++ /dev/null @@ -1,330 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#if MIGRAPHX_USE_MIOPEN -#include -#endif -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_REGISTER_OP(hip_allocate) -MIGRAPHX_REGISTER_OP(hip_fill) -MIGRAPHX_REGISTER_OP(hip_sync_stream) -MIGRAPHX_REGISTER_OP(hip_copy_to_gpu) -MIGRAPHX_REGISTER_OP(hip_copy_from_gpu) -MIGRAPHX_REGISTER_OP(hip_copy) -MIGRAPHX_REGISTER_OP(hip_allocate_memory) -MIGRAPHX_REGISTER_OP(hip_copy_literal) - -using hip_ptr = MIGRAPHX_MANAGE_PTR(void, hipFree); -using hip_host_ptr = MIGRAPHX_MANAGE_PTR(void, hipHostUnregister); - -std::string hip_error(int error) { return hipGetErrorString(static_cast(error)); } - -bool is_device_ptr(const void* ptr) -{ - hipPointerAttribute_t attr; - auto status = hipPointerGetAttributes(&attr, ptr); - if(status != hipSuccess) - return false; - return attr.type == hipMemoryTypeDevice; -} - -std::size_t get_available_gpu_memory() -{ - size_t free; - size_t total; - auto status = hipMemGetInfo(&free, &total); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed getting available memory: " + hip_error(status)); - return free; -} - -void* get_device_ptr(void* hptr) -{ - void* result = nullptr; - auto status = hipHostGetDevicePointer(&result, hptr, 0); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed getting device pointer: " + hip_error(status)); - return result; -} - -struct host_ptr_cache -{ - std::unordered_map> cache; - std::mutex m; - std::shared_ptr get(void* ptr) - { - std::lock_guard lock(m); - auto it = cache.find(ptr); - if(it != cache.end()) - return it->second.lock(); - return nullptr; - } - - void put(const std::shared_ptr& p) - { - std::lock_guard lock(m); - cache[p.get()] = p; - } -}; - -static host_ptr_cache& get_host_ptr_cache() -{ - static host_ptr_cache cache; - return cache; -} - -std::shared_ptr allocate_gpu(std::size_t sz, bool host = false) -{ - if(sz > get_available_gpu_memory()) - MIGRAPHX_THROW("Memory not available to allocate buffer: " + std::to_string(sz)); - void* alloc_ptr = nullptr; - auto status = host ? hipHostMalloc(&alloc_ptr, sz) : hipMalloc(&alloc_ptr, sz); - if(status != hipSuccess) - { - if(host) - MIGRAPHX_THROW("Gpu allocation failed: " + hip_error(status)); - else - return allocate_gpu(sz, true); - } - assert(alloc_ptr != nullptr); - std::shared_ptr result = share(hip_ptr{alloc_ptr}); - if(host) - { - get_host_ptr_cache().put(result); - } - return result; -} - -std::shared_ptr register_on_gpu(void* ptr, std::size_t sz) -{ - std::shared_ptr result = get_host_ptr_cache().get(ptr); - if(result) - { - return result; - } - auto status = hipHostRegister(ptr, sz, hipHostRegisterMapped); - if(status != hipSuccess) - MIGRAPHX_THROW("Gpu register failed: " + hip_error(status)); - result = share(hip_host_ptr{ptr}); - get_host_ptr_cache().put(result); - return result; -} - -template -std::vector read_from_gpu(const void* x, std::size_t sz) -{ - gpu_sync(); - std::vector result(sz); - assert(not is_device_ptr(result.data())); - if(not is_device_ptr(x)) - { - MIGRAPHX_THROW( - "read_from_gpu() requires Src buffer to be on the GPU, Copy from gpu failed\n"); - } - auto status = hipMemcpy(result.data(), x, sz * sizeof(T), hipMemcpyDeviceToHost); - if(status != hipSuccess) - MIGRAPHX_THROW("Copy from gpu failed: " + hip_error(status)); // NOLINT - return result; -} - -std::shared_ptr write_to_gpu(const void* x, std::size_t sz, bool host = false) -{ - gpu_sync(); - auto result = allocate_gpu(sz, host); - assert(is_device_ptr(result.get())); - assert(not is_device_ptr(x)); - auto status = hipMemcpy(result.get(), x, sz, hipMemcpyHostToDevice); - if(status != hipSuccess) - MIGRAPHX_THROW("Copy to gpu failed: " + hip_error(status)); - return result; -} - -template -hip_ptr write_to_gpu(const T& x) -{ - using type = typename T::value_type; - auto size = x.size() * sizeof(type); - return write_to_gpu(x.data(), size); -} - -argument allocate_gpu(const shape& s, bool host) -{ - auto p = allocate_gpu(s.bytes() + 1, host); - return {s, [p]() mutable { return reinterpret_cast(p.get()); }}; -} - -argument register_on_gpu(const argument& arg) -{ - auto arg_shared = arg.share(); - auto p = register_on_gpu(arg_shared.data(), arg_shared.get_shape().bytes()); - auto s = arg_shared.get_shape(); - return {s, [p, a = std::move(arg_shared)]() mutable { return get_device_ptr(p.get()); }}; -} - -argument to_gpu(const argument& arg, bool host) -{ - argument result; - arg.visit( - [&](auto x) { - auto p = write_to_gpu(arg.data(), arg.get_shape().bytes(), host); - result = {x.get_shape(), p}; - }, - [&](const auto& xs) { - std::vector args; - std::transform(xs.begin(), xs.end(), std::back_inserter(args), [&](auto x) { - return to_gpu(x, host); - }); - result = argument{args}; - }); - return result; -} - -argument from_gpu(const argument& arg) -{ - argument result; - arg.visit( - [&](auto x) { - using type = typename decltype(x)::value_type; - auto v = read_from_gpu(arg.data(), x.get_shape().bytes() / sizeof(type)); - // cppcheck-suppress returnDanglingLifetime - result = {x.get_shape(), [v]() mutable { return v.data(); }}; - }, - [&](const auto& xs) { - std::vector args; - std::transform(xs.begin(), xs.end(), std::back_inserter(args), [&](auto x) { - return from_gpu(x); - }); - result = argument{args}; - }); - - return result; -} - -void set_device(std::size_t id) -{ - auto status = hipSetDevice(id); - if(status != hipSuccess) - MIGRAPHX_THROW("Error setting device"); -} - -void gpu_sync() -{ - auto status = hipDeviceSynchronize(); - if(status != hipSuccess) - MIGRAPHX_THROW("hip device synchronization failed: " + hip_error(status)); -} - -void gpu_sync(const context& ctx) { ctx.finish(); } - -void hip_async_memset(context& ctx, const argument& dst, int value) -{ - std::size_t dst_size = dst.get_shape().bytes(); - auto status = hipMemsetAsync(dst.data(), value, dst_size, ctx.get_stream().get()); - if(status != hipSuccess) - MIGRAPHX_THROW("Gpu fill failed: " + hip_error(status)); -} - -void hip_async_copy(context& ctx, const argument& src, const argument& dst, hipMemcpyKind kind) -{ - std::size_t src_size = src.get_shape().bytes(); - std::size_t dst_size = dst.get_shape().bytes(); - if(src_size > dst_size) - MIGRAPHX_THROW("Not enough memory available in destination to do copy"); - auto status = hipMemcpyAsync(dst.data(), src.data(), src_size, kind, ctx.get_stream().get()); - if(status != hipSuccess) - MIGRAPHX_THROW("Gpu copy failed: " + hip_error(status)); -} - -void gpu_copy(context& ctx, const argument& src, const argument& dst) -{ - // Workaround: Use contiguous as hip's memcpy is broken - device::contiguous(ctx.get_stream().get(), dst, src); - // hip_async_copy(ctx, src, dst, hipMemcpyDeviceToDevice); -} - -void copy_to_gpu(context& ctx, const argument& src, const argument& dst) -{ - if(src.get_shape() == dst.get_shape() and dst.get_shape().packed()) - { - hip_async_copy(ctx, src, dst, hipMemcpyHostToDevice); - } - else - { - gpu_copy(ctx, register_on_gpu(src), dst); - } -} - -void copy_from_gpu(context& ctx, const argument& src, const argument& dst) -{ - if(src.get_shape() == dst.get_shape() and dst.get_shape().packed()) - { - hip_async_copy(ctx, src, dst, hipMemcpyDeviceToHost); - } - else - { - gpu_copy(ctx, src, register_on_gpu(dst)); - } -} - -argument get_preallocation(context& ctx, const std::string& id) -{ - return ctx.get_current_device().preallocations.at(id); -} - -void gpu_fill(context& ctx, const argument& dst, int value) -{ - if(dst.get_sub_objects().empty()) - { - // TODO: Handle non-packed tensor when value is not 0 - assert(dst.get_shape().packed() and value == 0); - hip_async_memset(ctx, dst, value); - } - else - { - for(const auto& arg : dst.get_sub_objects()) - gpu_fill(ctx, arg, value); - } -} - -void store_preallocated_param(context& ctx, const std::string& id, const argument& a) -{ - ctx.get_current_device().preallocations[id] = a; -} - -// clang-format off -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/hip_gemm_impl.cpp b/docker/rocm/migraphx/targets/gpu/hip_gemm_impl.cpp deleted file mode 100644 index 966927da7..000000000 --- a/docker/rocm/migraphx/targets/gpu/hip_gemm_impl.cpp +++ /dev/null @@ -1,754 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#if MIGRAPHX_USE_HIPBLASLT -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using microseconds = std::chrono::duration; - -hipDataType compute_to_hip_type(hipblasComputeType_t type) -{ - switch(type) - { - case HIPBLAS_COMPUTE_32F: return HIP_R_32F; - case HIPBLAS_COMPUTE_32I: return HIP_R_32I; - case HIPBLAS_COMPUTE_16F: - case HIPBLAS_COMPUTE_64F: - case HIPBLAS_COMPUTE_32I_PEDANTIC: - case HIPBLAS_COMPUTE_16F_PEDANTIC: - case HIPBLAS_COMPUTE_32F_PEDANTIC: - case HIPBLAS_COMPUTE_64F_PEDANTIC: - case HIPBLAS_COMPUTE_32F_FAST_16F: - case HIPBLAS_COMPUTE_32F_FAST_16BF: - case HIPBLAS_COMPUTE_32F_FAST_TF32: - MIGRAPHX_THROW("HIPBLAS_GEMM: conversion from hipComputeType_t to hipDataType failed"); - } -} - -// Convert hipBLAS datatypes to equivalent MIGraphX data types -hipDataType get_type_hipblas(shape::type_t type) -{ - switch(type) - { - case shape::double_type: return HIP_R_64F; - case shape::float_type: return HIP_R_32F; - case shape::half_type: return HIP_R_16F; - case shape::int8_type: return HIP_R_8I; - case shape::uint8_type: return HIP_R_8U; - case shape::int32_type: return HIP_R_32I; - case shape::uint32_type: return HIP_R_32U; - case shape::fp8e4m3fnuz_type: return HIP_R_8F_E4M3_FNUZ; - case shape::fp8e5m2fnuz_type: - return HIP_R_8F_E5M2_FNUZ; -// TODO can remove this preprocessor conditional when hip verison defaults to have these types -#ifdef ROCM_USE_FLOAT8 - case shape::fp8e4m3fn_type: return HIP_R_8F_E4M3; - case shape::fp8e5m2_type: return HIP_R_8F_E5M2; -#else - case shape::fp8e4m3fn_type: - case shape::fp8e5m2_type: -#endif - case shape::tuple_type: - case shape::bool_type: - case shape::uint16_type: - case shape::int16_type: - case shape::int64_type: - case shape::uint64_type: MIGRAPHX_THROW("HIPBLAS_GEMM: data type not supported!"); - case shape::bf16_type: return HIP_R_16BF; - } - - MIGRAPHX_THROW("HIPBLAS_GEMM: data type not supported!"); -} - -void blas_shape_hip(const shape& in_shape) -{ - if(in_shape.lens().size() < 2) - return; - auto s = in_shape.normalize_standard(); - if(std::none_of(s.strides().end() - 2, s.strides().end(), [](auto i) { return i == 1; })) - MIGRAPHX_THROW("GPU_GEMM: needs to have one matrix stride as 1"); - if(std::any_of(s.strides().end() - 2, s.strides().end(), [](auto i) { return i == 0; })) - MIGRAPHX_THROW("GPU_GEMM: matrix dimensions can't be broadcasted"); - if(s.lens().size() < 3) - return; - shape batch_shape{s.type(), - {s.lens().begin(), s.lens().end() - 2}, - {s.strides().begin(), s.strides().end() - 2}}; - auto batch_shapes = reduce_dims({batch_shape}); - if(batch_shapes.front().lens().size() != 1) - MIGRAPHX_THROW("GPU_GEMM: Batch dimension is not collapsible"); -} - -shape transpose_batch_hip(const shape& s, unsigned trans_batch) -{ - if(trans_batch == 0) - return s; - if(s.lens().size() < 3) - return s; - auto batch = s.lens().size() - 3; - std::vector perm(s.lens().size()); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[batch], perm[batch + trans_batch]); - return shape::from_permutation(s.type(), s.lens(), perm); -} - -static bool is_transposed_hip(const shape& s) { return s.transposed() and s.strides().back() != 1; } - -static int32_t get_batch_stride_hip(const shape& s) -{ - // This value is not needed for non-strided inputs - if(s.strides().size() < 3) - return 0; - else - return s.strides()[s.strides().size() - 3]; -} - -/** - * Wrapper for multiple hipBLASLt calls. The constructor creates parameters for - * these calls based on data shapes and other values contained in the associated - * instruction and operation. - */ -struct hip_gemm_impl -{ - hip_gemm_impl(const shape& output_shape, - const std::vector& input_shapes, - float alpha_param, - float beta_param) - : alpha(alpha_param), beta(beta_param), is_3inputs(input_shapes.size() == 5) - { - if(not is_3inputs) - { - beta = 0; - } - - // Create lambdas that will cast alpha, beta to the output shape's type - // and retain the values being pointed to - output_shape.visit_type([&](auto as) { - if(as.is_integral()) - { - int32_t alpha_r = int32_t(alpha); - int32_t beta_r = int32_t(beta); - get_alpha = [=] { return &alpha_r; }; - get_beta = [=] { return &beta_r; }; - } - else - { - get_alpha = [=] { return α }; - get_beta = [=] { return β }; - } - }); - - transa = is_transposed_hip(input_shapes[0]); - transb = is_transposed_hip(input_shapes[1]); - op_a = transa ? HIPBLAS_OP_T : HIPBLAS_OP_N; - op_b = transb ? HIPBLAS_OP_T : HIPBLAS_OP_N; - - auto n_dim = output_shape.lens().size(); - auto dim_0 = n_dim - 2; - auto dim_1 = n_dim - 1; - // Leading dimensions of matrices - lda = input_shapes[0].strides()[transa ? dim_1 : dim_0]; - ldb = input_shapes[1].strides()[transb ? dim_1 : dim_0]; - ldc = is_3inputs ? input_shapes[2].strides()[dim_0] : input_shapes[3].strides()[dim_0]; - ldd = is_3inputs ? input_shapes[4].strides()[dim_0] : ldc; - - auto out_lens = output_shape.lens(); - m = out_lens[dim_0]; - n = out_lens[dim_1]; - k = input_shapes[0].lens()[dim_1]; - - a_stride = get_batch_stride_hip(input_shapes[0]); - b_stride = get_batch_stride_hip(input_shapes[1]); - c_stride = is_3inputs ? get_batch_stride_hip(input_shapes[2]) - : get_batch_stride_hip(input_shapes[3]); - d_stride = is_3inputs ? get_batch_stride_hip(input_shapes[4]) : c_stride; - num_matrices = std::accumulate( - out_lens.rbegin() + 2, out_lens.rend(), std::size_t{1}, std::multiplies()); - - arg_type = get_type_hipblas(input_shapes[0].type()); - output_type = is_3inputs ? get_type_hipblas(input_shapes[4].type()) - : get_type_hipblas(input_shapes[3].type()); - - if(arg_type == HIP_R_8I or arg_type == HIP_R_8U) - { - compute_type = HIPBLAS_COMPUTE_32I; - } - else - { - compute_type = HIPBLAS_COMPUTE_32F; - } - if(op_a == HIPBLAS_OP_T) - { - hipblaslt_invoke( - [&]() { return hipblasLtMatrixLayoutCreate(&mat_a, arg_type, m, k, lda); }); - } - else - { - hipblaslt_invoke( - [&]() { return hipblasLtMatrixLayoutCreate(&mat_a, arg_type, k, m, lda); }); - } - if(op_b == HIPBLAS_OP_T) - { - hipblaslt_invoke( - [&]() { return hipblasLtMatrixLayoutCreate(&mat_b, arg_type, k, n, ldb); }); - } - else - { - hipblaslt_invoke( - [&]() { return hipblasLtMatrixLayoutCreate(&mat_b, arg_type, n, k, ldb); }); - } - hipblaslt_invoke( - [&]() { return hipblasLtMatrixLayoutCreate(&mat_c, output_type, n, m, ldc); }); - - if(is_3inputs) - { - hipblaslt_invoke( - [&]() { return hipblasLtMatrixLayoutCreate(&mat_d, output_type, n, m, ldd); }); - } - if(num_matrices > 1) - { - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute(mat_a, - HIPBLASLT_MATRIX_LAYOUT_BATCH_COUNT, - &num_matrices, - sizeof(num_matrices)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute(mat_b, - HIPBLASLT_MATRIX_LAYOUT_BATCH_COUNT, - &num_matrices, - sizeof(num_matrices)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute(mat_c, - HIPBLASLT_MATRIX_LAYOUT_BATCH_COUNT, - &num_matrices, - sizeof(num_matrices)); - }); - - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute( - mat_a, - HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, - &a_stride, - sizeof(a_stride)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute( - mat_b, - HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, - &b_stride, - sizeof(b_stride)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute( - mat_c, - HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, - &c_stride, - sizeof(c_stride)); - }); - - if(is_3inputs) - { - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute(mat_d, - HIPBLASLT_MATRIX_LAYOUT_BATCH_COUNT, - &num_matrices, - sizeof(num_matrices)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatrixLayoutSetAttribute( - mat_d, - HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, - &d_stride, - sizeof(d_stride)); - }); - } - } - hipblaslt_invoke([&]() { - return hipblasLtMatmulDescCreate( - &hipblaslt_desc, compute_type, compute_to_hip_type(compute_type)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatmulDescSetAttribute( - hipblaslt_desc, HIPBLASLT_MATMUL_DESC_TRANSB, &op_a, sizeof(int32_t)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatmulDescSetAttribute( - hipblaslt_desc, HIPBLASLT_MATMUL_DESC_TRANSA, &op_b, sizeof(int32_t)); - }); - - // Transfer ownership of raw pointers to managed pointers. - managed_hipblaslt_desc.reset(hipblaslt_desc); - managed_mat_a.reset(mat_a); - managed_mat_b.reset(mat_b); - managed_mat_c.reset(mat_c); - if(is_3inputs) - { - managed_mat_d.reset(mat_d); - } - } - - ~hip_gemm_impl() {} - - struct solution - { - solution() : handle(nullptr), preference(nullptr) {} - - auto get_hipblaslt_preference() - { - if(hbltpreference == nullptr) - { - hbltpreference = create_hipblaslt_preference_ptr(); - } - assert(hbltpreference.get() != nullptr); - return hbltpreference.get(); - } - - void init(context& ctx) - { - if(handle == nullptr) - { - handle = ctx.get_stream().get_hipblaslt(); - preference = get_hipblaslt_preference(); - } - } - - auto& get_result(context& ctx, hip_gemm_impl& gemm, int32_t idx) - { - init(ctx); - if(idx == 0) - { - // use default solution - const int n_sol = 1; - int returned_algo_count; - heuristic_result.resize(n_sol); - uint64_t max_workspace = std::numeric_limits::max(); - hipblaslt_invoke([&]() { - return hipblasLtMatmulPreferenceSetAttribute( - preference, - HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, - &max_workspace, - sizeof(uint64_t)); - }); - hipblaslt_invoke([&]() { - return hipblasLtMatmulAlgoGetHeuristic(handle, - gemm.hipblaslt_desc, - gemm.mat_b, - gemm.mat_a, - gemm.mat_c, - gemm.is_3inputs ? gemm.mat_d - : gemm.mat_c, - preference, - n_sol, - heuristic_result.data(), - &returned_algo_count); - }); - - if(returned_algo_count != n_sol) - { - std::cout << "less solution found! request: " << n_sol - << ", found: " << returned_algo_count << std::endl; - } - } - else - { - // query for the solutions. 1st as the best. - std::vector algo_index = {idx}; - hipblaslt_invoke([&]() { - return hipblaslt_ext::getAlgosFromIndex(handle, algo_index, heuristic_result); - }); - assert(heuristic_result.size() == 1); - } - return heuristic_result; - } - - private: - hipblasLtHandle_t handle; - hipblasLtMatmulPreference_t preference; - std::vector heuristic_result; - shared hbltpreference = nullptr; - } solution; - - /** - * Helper method to create that subset of a long hipblaslt argument list that is common - * to multiple "hipblasLtMatmul" calls. - * - * The hipblaslt GEMM API handles inputs and output matrices as - * column-major format. When doing a C = A * B, we actually do - * C^T = (B^T) * (A^T). That is the reason we input args[1] as - * A and args[0] as B in calling the hipblaslt GEMM. - * - * */ - auto create_hipblaslt_args_common(context& ctx, - const std::vector& args, - int32_t solution_idx) - { - auto* algo = &solution.get_result(ctx, *this, solution_idx)[0].algo; - size_t workspace_size = ((is_3inputs ? args[3] : args[2]).get_shape()).bytes(); - return pack(ctx.get_stream().get_hipblaslt(), - hipblaslt_desc, - get_alpha(), // alpha - args[1].data(), // A - mat_b, // Adesc - args[0].data(), // B - mat_a, // Bdesc - get_beta(), // beta - is_3inputs ? args[2].data() : args[3].data(), // C - mat_c, // Cdesc - is_3inputs ? args[4].data() : args[3].data(), // D - is_3inputs ? mat_d : mat_c, // Ddesc - algo, // algo - is_3inputs ? args[3].data() : args[2].data(), // workspace - workspace_size, // workspaceSizeInBytes - ctx.get_stream().get() // stream - ); - } - - auto create_hipblaslt_supporting_args_common(context& ctx, - const std::vector& args, - hipblasLtMatmulAlgo_t& algo, - size_t& workspace_size) const - { - (void)(args); - return pack(ctx.get_stream().get_hipblaslt(), - hipblaslt_desc, - get_alpha(), - mat_b, - mat_a, - get_beta(), - mat_c, - is_3inputs ? mat_d : mat_c, - algo, - workspace_size); - } - - void - run(context& ctx, const std::vector& input_args, int32_t solution_idx = 0) // const - { - auto common_args = create_hipblaslt_args_common(ctx, input_args, solution_idx); - hipblaslt_invoke(&hipblasLtMatmul, common_args); - } - - auto - validate(context& ctx, const std::vector& input_shapes, int32_t solution_idx) // const - { - // Create dummy arguments for the shapes, and call the overloaded method - std::vector input_args; - std::transform(input_shapes.begin(), - input_shapes.end(), - std::back_inserter(input_args), - [](const shape& x) { return to_gpu(generate_argument(x)); }); - - return validate(ctx, input_args, solution_idx); - } - - /** - * Checks a particular solution for validity by running it (could be invalid if this model was - * tuned with a different hipBLASLt version) - * - * @return Returns either solution_idx if valid, or else the default value 0 - * if not. The default does not mean list index 0, but tells the picker - * to choose a solution. - */ - int32_t - validate(context& ctx, const std::vector& input_args, int32_t solution_idx) // const - { - auto common_args = create_hipblaslt_args_common(ctx, input_args, solution_idx); - auto check_valid = hipblaslt_invoke(&hipblasLtMatmul, common_args, false); - if(check_valid != HIPBLAS_STATUS_SUCCESS) - { - std::cerr << "WARNING: tuned solution is invalid; reverting to default" << std::endl; - return 0; - } - return solution_idx; - } - - /** - * Get workspace size for the solution index: Gets algo from the solution index, - * and calls matmulIsAlgoSupported() to get the workspace size. - */ - - size_t get_workspace_size(context& ctx, - const std::vector& input_shapes, - int32_t solution_idx) const - { - size_t workspace_size = hipblaslt_workspace_size; - std::vector input_args; - std::transform(input_shapes.begin(), - input_shapes.end(), - std::back_inserter(input_args), - [](const shape& x) { return to_gpu(generate_argument(x)); }); - - std::vector algo_index = {solution_idx}; - std::vector heuristic_result; - - hipblaslt_invoke([&]() { - return hipblaslt_ext::getAlgosFromIndex( - ctx.get_stream().get_hipblaslt(), algo_index, heuristic_result); - }); - assert(heuristic_result.size() == 1); - - auto algo = heuristic_result[0].algo; - size_t ret_workspace_size = 0; - auto supporting_args = - create_hipblaslt_supporting_args_common(ctx, input_args, algo, ret_workspace_size); - - auto status = - hipblaslt_invoke(&hipblaslt_ext::matmulIsAlgoSupported, supporting_args, false); - - // If algo is supported, update the workspace size to the actual size needed. - // Otherwise, use the default workspace size. - if(status == HIPBLAS_STATUS_SUCCESS) - { - // TODO: Remove this check once issues with '0' workspace size are resolved. - // Temporarily, we use the approach where, if the returned workspace size is '0', - // we use the default workspace size. - // Otherwise, we use the returned workspace size. - if(ret_workspace_size != 0) - workspace_size = ret_workspace_size; - } - return workspace_size; - } - - /** - * Find best hipBLASLt solution: Get list of solutions and try them all, returning the index - * of the fastest one. - */ - int tune(context& ctx, const std::vector& input_shapes) // const - { - // tuning meta parameters - const int hot_calls = 40; - - std::vector input_args; - std::transform(input_shapes.begin(), - input_shapes.end(), - std::back_inserter(input_args), - [](const shape& x) { return to_gpu(generate_argument(x)); }); - - std::vector result; - hipblaslt_invoke([&]() { - return hipblaslt_ext::getAllAlgos(ctx.get_stream().get_hipblaslt(), - hipblaslt_ext::GemmType::HIPBLASLT_GEMM, - op_a, - op_b, - arg_type, - arg_type, - output_type, - output_type, - compute_type, - result); - }); - std::vector solution_indices; - int returned_algo_count = result.size(); - for(int i = 0; i < returned_algo_count; i++) - { - auto algo = result[i].algo; - size_t ret_workspace_size = 0; - auto supporting_args = - create_hipblaslt_supporting_args_common(ctx, input_args, algo, ret_workspace_size); - try - { - hipblaslt_invoke(&hipblaslt_ext::matmulIsAlgoSupported, supporting_args); - solution_indices.push_back(hipblaslt_ext::getIndexFromAlgo(algo)); - } - catch(...) - { - // algo is not supported, continue in that case - continue; - } - } - - double best_time = std::numeric_limits::max(); - double first_time = -1; - - // Initialize to default solution index - int32_t best_sol = 0; - // If no valid/supported solution is returned, use hipblasLtMatmulAlgoGetHeuristic - // to get an algo and use solution index from that algo. - if(solution_indices.empty()) - { - auto algo = solution.get_result(ctx, *this, 0)[0].algo; - solution_indices.push_back(hipblaslt_ext::getIndexFromAlgo(algo)); - } - for(auto sol : solution_indices) - { - // Warmup: the first call to an op. may not be representative since there is - // more time taken initializing caches, etc. so we won't time it. - run(ctx, input_args, sol); - double host_time = time([&] { - for([[maybe_unused]] int hc : range(hot_calls)) - run(ctx, input_args, sol); - ctx.finish(); - }); - - host_time /= hot_calls; - - // dev/evaluation only: track time for first solution. - if(first_time < 0) - first_time = host_time; - - // track current best - if(host_time < best_time) - { - best_sol = sol; - best_time = host_time; - } - } - - std::cout << "Winning GEMM solution: " << best_sol << " in " << best_time << " ms, beats " - << first_time << "ms" << std::endl; - return best_sol; - } - - // hipblaslt - size_t num_matrices = 0; - uint64_t m = 0; - uint64_t n = 0; - uint64_t k = 0; - bool transa = false; - bool transb = false; - float alpha = 0; - float beta = 0; - std::function get_alpha{}; - std::function get_beta{}; - - int64_t lda = 0; - int64_t ldb = 0; - int64_t ldc = 0; - int64_t ldd = 0; - int64_t a_stride = 0; - int64_t b_stride = 0; - int64_t c_stride = 0; - int64_t d_stride = 0; - bool is_3inputs = true; - - hipDataType arg_type = HIP_R_32F; - hipblasComputeType_t compute_type = HIPBLAS_COMPUTE_32F; - hipDataType output_type = HIP_R_32F; - hipblasLtMatmulDesc_t hipblaslt_desc; - hipblasOperation_t op_a; - hipblasOperation_t op_b; - using hipblaslt_matrix_layout = MIGRAPHX_MANAGE_PTR(hipblasLtMatrixLayout_t, - hipblasLtMatrixLayoutDestroy); - using hipblaslt_mat_mul_desc = MIGRAPHX_MANAGE_PTR(hipblasLtMatmulDesc_t, - hipblasLtMatmulDescDestroy); - hipblaslt_matrix_layout managed_mat_a, managed_mat_b, managed_mat_c, managed_mat_d; - hipblaslt_mat_mul_desc managed_hipblaslt_desc; - hipblasLtMatrixLayout_t mat_a, mat_b, mat_c, mat_d; - hipblasLtHandle_t handle; - hipblasLtMatmulPreference_t preference; -}; // hip_gemm_impl - -void hip_gemm_compute(context& ctx, - const shape& output_shape, - const std::vector& args, - float alpha, - float beta, - int32_t solution_idx) -{ - std::vector input_shapes; - std::transform(args.begin(), - args.end(), - std::back_inserter(input_shapes), - [](const argument& x) { return x.get_shape().normalize_standard(); }); - auto gemm_item = hip_gemm_impl(output_shape, input_shapes, alpha, beta); - gemm_item.run(ctx, args, solution_idx); -} - -static value hip_gemm_problem(const shape& output_shape, std::vector input_shapes) -{ - input_shapes.push_back(output_shape); - return to_value(input_shapes); -} - -static void hip_gemm_save_solution(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - int32_t solution_idx) -{ - ctx.get_problem_cache().insert( - "hipblaslt", hip_gemm_problem(output_shape, input_shapes), solution_idx); -} - -int32_t hip_gemm_finalize(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - float alpha, - float beta, - int32_t solution_idx) -{ - auto gemm_item = hip_gemm_impl(output_shape, input_shapes, alpha, beta); - if(solution_idx == 0) - { - solution_idx = gemm_item.tune(ctx, input_shapes); - hip_gemm_save_solution(ctx, output_shape, input_shapes, solution_idx); - } - // If a tuned solution index is already given, don't tune again but validate - // in case the data was tuned with a different hipBLASLt version. - else - { - solution_idx = gemm_item.validate(ctx, input_shapes, solution_idx); - } - return solution_idx; -} - -int32_t hip_gemm_default_solution(context& ctx, - const shape& output_shape, - const std::vector& input_shapes) -{ - auto sol = - ctx.get_problem_cache().get("hipblaslt", hip_gemm_problem(output_shape, input_shapes)); - if(sol.has_value()) - return sol->to(); - return 0; -} - -size_t hip_gemm_workspace_size(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - float alpha, - float beta, - int32_t solution_idx) -{ - auto gemm_item = hip_gemm_impl(output_shape, input_shapes, alpha, beta); - return gemm_item.get_workspace_size(ctx, input_shapes, solution_idx); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_USE_HIPBLASLT diff --git a/docker/rocm/migraphx/targets/gpu/hipblaslt.cpp b/docker/rocm/migraphx/targets/gpu/hipblaslt.cpp deleted file mode 100644 index 47a9e9273..000000000 --- a/docker/rocm/migraphx/targets/gpu/hipblaslt.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_HIPBLASLT -// for hipblaslt only -static const size_t workspace_size = hipblaslt_workspace_size; - -hipblaslt_handle_ptr create_hipblaslt_handle_ptr() -{ - hipblasLtHandle_t handle; - hipblasLtCreate(&handle); - return hipblaslt_handle_ptr{handle}; -} - -hipblaslt_preference_ptr create_hipblaslt_preference_ptr() -{ - hipblasLtMatmulPreference_t preference; - hipblasLtMatmulPreferenceCreate(&preference); - hipblaslt_invoke([&]() { - return hipblasLtMatmulPreferenceSetAttribute(preference, - HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, - &workspace_size, - sizeof(workspace_size)); - }); - return hipblaslt_preference_ptr{preference}; -} - -bool hipblaslt_supported() -{ - const auto device_name = trim(split_string(get_device_name(), ':').front()); - // hipblaslt is supported for MI200 and above, and Navi3x and above. - return (device_name == "gfx90a" or - (starts_with(device_name, "gfx94") and device_name >= "gfx940") or - starts_with(device_name, "gfx110") or starts_with(device_name, "gfx120")); -} - -#endif // MIGRAPHX_USE_HIPBLASLT - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/hiprtc/CMakeLists.txt b/docker/rocm/migraphx/targets/gpu/hiprtc/CMakeLists.txt deleted file mode 100644 index a8cb3cec0..000000000 --- a/docker/rocm/migraphx/targets/gpu/hiprtc/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -add_executable(migraphx-hiprtc-driver - main.cpp -) -rocm_clang_tidy_check(migraphx-hiprtc-driver) -# On Windows, the driver's default 1MB stack size is not enough - increasing to 4MB. -set(STACK_SIZE 4194304) -if(MSVC) - target_link_options(migraphx-hiprtc-driver PRIVATE /STACK:${STACK_SIZE}) -elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_SIMULATE_ID MATCHES "MSVC") - target_link_options(migraphx-hiprtc-driver PRIVATE -Xlinker /stack:${STACK_SIZE}) -endif() -target_link_libraries(migraphx-hiprtc-driver PRIVATE migraphx_gpu) -add_dependencies(migraphx_all_targets migraphx-hiprtc-driver) -rocm_install_targets( - TARGETS migraphx-hiprtc-driver -) diff --git a/docker/rocm/migraphx/targets/gpu/hiprtc/main.cpp b/docker/rocm/migraphx/targets/gpu/hiprtc/main.cpp deleted file mode 100644 index d443ce49e..000000000 --- a/docker/rocm/migraphx/targets/gpu/hiprtc/main.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#endif - -std::vector read_stdin() -{ -#ifdef _WIN32 - // Set stream translation mode to BINARY to suppress translations. - // https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/setmode?view=msvc-170 - auto old_mode = _setmode(_fileno(stdin), _O_BINARY); - if(old_mode == -1) - MIGRAPHX_THROW(std::strerror(errno)); -#endif - std::vector result; - std::array buffer{}; - std::size_t len = 0; - while((len = std::fread(buffer.data(), 1, buffer.size(), stdin)) > 0) - { - if(std::ferror(stdin) != 0 and std::feof(stdin) == 0) - MIGRAPHX_THROW(std::strerror(errno)); - - result.insert(result.end(), buffer.data(), buffer.data() + len); - } -#ifdef _WIN32 - // Reset to the previously set translation mode. - _setmode(_fileno(stdin), old_mode); -#endif - return result; -} - -int main(int argc, char const* argv[]) -{ - if(argc < 2 or migraphx::contains({"-h", "--help", "-v", "--version"}, std::string(argv[1]))) - { - std::cout << "USAGE:" << std::endl; - std::cout << " "; - std::cout << "Used internally by migraphx to compile hip programs out-of-process." - << std::endl; - std::exit(0); - } - std::string output_name = argv[1]; - try - { - auto v = migraphx::from_msgpack(read_stdin()); - std::vector srcs; - migraphx::from_value(v.at("srcs"), srcs); - auto out = - migraphx::gpu::compile_hip_src_with_hiprtc(std::move(srcs), - v.at("params").to_vector(), - v.at("arch").to()); - if(not out.empty()) - migraphx::write_buffer(output_name, out.front()); - } - catch(const std::exception& err) - { - std::cout << err.what() << std::endl; - } -} diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/abs.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/abs.hpp deleted file mode 100644 index 1a9f4b878..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/abs.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ABS_HPP -#define MIGRAPHX_GUARD_RTGLIB_ABS_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; -#if MIGRAPHX_USE_MIOPEN - -struct miopen_abs -{ - op::abs op; - shared ad; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::abs"; } - shape compute_shape(const std::vector& inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - void finalize(context&, const shape&, const std::vector&); - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/allocation_model.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/allocation_model.hpp deleted file mode 100644 index 249901f23..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/allocation_model.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_GPU_ALLOCATION_MODEL_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_GPU_ALLOCATION_MODEL_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct MIGRAPHX_GPU_EXPORT gpu_allocation_model -{ - std::string name() const; - std::string copy() const; - operation allocate(const shape& s) const; - operation preallocate(const shape& s, const std::string& id) const; - bool needs_out_params() const { return true; } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/analyze_streams.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/analyze_streams.hpp deleted file mode 100644 index cf3d2ee42..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/analyze_streams.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_ANALYZE_STREAMS_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_ANALYZE_STREAMS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace gpu { - -MIGRAPHX_GPU_EXPORT std::vector analyze_streams(const module& m); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/argmax.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/argmax.hpp deleted file mode 100644 index e05678fa8..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/argmax.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ARGMAX_HPP -#define MIGRAPHX_GUARD_RTGLIB_ARGMAX_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_argmax -{ - op::argmax op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::argmax"; } - shape compute_shape(const std::vector& inputs) const; - argument compute(context& ctx, const shape&, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/argmin.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/argmin.hpp deleted file mode 100644 index 071eb525e..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/argmin.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_ARGMIN_HPP -#define MIGRAPHX_GUARD_RTGLIB_ARGMIN_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_argmin -{ - op::argmin op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::argmin"; } - shape compute_shape(const std::vector& inputs) const; - argument compute(context& ctx, const shape&, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/ck.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/ck.hpp deleted file mode 100644 index 18d4dce25..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/ck.hpp +++ /dev/null @@ -1,165 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_CK_HPP -#define MIGRAPHX_GUARD_GPU_CK_HPP - -#include -#include -#include -#include -#include - -#include "ck/host/device_gemm_multiple_d.hpp" -#include "ck/host/device_batched_gemm_softmax_gemm.hpp" - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -#ifndef _WIN32 -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_CK); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_LOG_CK_GEMM); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_CK_DEBUG); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TUNE_CK); -#endif - -// NOLINTNEXTLINE -const char* const disable_warning_pragma = R"__migraphx__( -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Weverything" -${content} -#pragma clang diagnostic pop -)__migraphx__"; - -template -std::string ck_disable_warnings(P p) -{ - return interpolate_string(disable_warning_pragma, - {{"content", std::string{p.data(), p.size()}}}); -} - -static std::unordered_map create_ck_header_strings() -{ - std::unordered_map result; - auto ck_headers = ck::host::GetHeaders(); - - std::transform( - ck_headers.begin(), ck_headers.end(), std::inserter(result, result.begin()), [&](auto& p) { - return std::pair(p.first, ck_disable_warnings(p.second)); - }); - return result; -} - -static std::vector create_ck_headers() -{ - static const auto& header_strings = create_ck_header_strings(); - std::vector srcs; - std::transform(header_strings.begin(), - header_strings.end(), - std::back_inserter(srcs), - [&](auto& p) { return src_file{p}; }); - return srcs; -} - -static inline const std::vector& ck_headers() -{ - static const auto& headers = create_ck_headers(); - return headers; -} - -inline bool transposed_matrix(const shape& s) { return s.strides().back() != 1; } - -inline ck::host::DataType get_type(const shape& s) -{ - if(s.type() == shape::half_type) - return ck::host::DataType::Half; - else if(s.type() == shape::float_type) - return ck::host::DataType::Float; - else if(s.type() == shape::int8_type) - return ck::host::DataType::Int8; - else if(s.type() == shape::int32_type) - return ck::host::DataType::Int32; - MIGRAPHX_THROW("Unsupported ck type"); -} - -inline std::size_t get_batch_count(const shape& s) -{ - return std::accumulate( - s.lens().rbegin() + 2, s.lens().rend(), std::size_t{1}, std::multiplies()); -} - -inline void fold_batch_dims(shape& s) -{ - auto lens = s.lens(); - if(lens.size() <= 2) - return; - auto batch_count = get_batch_count(s); - auto m1 = lens.at(lens.size() - 2); - auto m2 = lens.at(lens.size() - 1); - if(transposed_matrix(s)) - s = shape{s.type(), {m1, m2 * batch_count}}; - else - s = shape{s.type(), {m1 * batch_count, m2}}; -} - -inline void remove_batch_dims(shape& s) -{ - auto lens = s.lens(); - if(lens.size() <= 2) - return; - auto m1 = lens.at(lens.size() - 2); - auto m2 = lens.at(lens.size() - 1); - s = shape{s.type(), {m1, m2}}; -} - -inline bool standard_batch(const shape& s) -{ - if(s.lens().size() < 3) - return true; - std::vector lens(s.lens().begin(), s.lens().end() - 2); - std::vector strides(s.strides().begin(), s.strides().end() - 2); - auto base = *(s.lens().end() - 2) * *(s.lens().end() - 1); - std::transform(strides.begin(), strides.end(), strides.begin(), [&](auto stride) { - return stride / base; - }); - return shape{s.type(), lens, strides}.standard(); -} - -inline bool can_fold_batch(const std::vector& inputs) -{ - const auto& b_shape = inputs[1]; - if(std::any_of(inputs.begin() + 2, inputs.end() - 1, [](auto input) { - return not standard_batch(input); - })) - return false; - const auto& b_strides = b_shape.strides(); - return std::all_of( - b_strides.begin(), b_strides.end() - 2, [](auto stride) { return stride == 0; }); -} - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_CK_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/code_object_op.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/code_object_op.hpp deleted file mode 100644 index 818676728..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/code_object_op.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_CODE_OBJECT_OP_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_CODE_OBJECT_OP_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct code_object_op -{ - value::binary code_object{}; - std::string symbol_name = ""; - std::size_t global = 0; - std::size_t local = 0; - std::vector expected_inputs{}; - shape output{}; - std::int64_t output_arg = -1; - kernel k{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.code_object, "code_object"), - f(self.symbol_name, "symbol_name"), - f(self.global, "global"), - f(self.local, "local"), - f(self.expected_inputs, "expected_inputs"), - f(self.output, "output"), - f(self.output_arg, "output_arg")); - } - - value attributes() const { return {{"group", group()}}; } - - std::string group() const { return "gpu::code_object::" + symbol_name; } - - std::string name() const { return "gpu::code_object"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - void finalize(context&, const shape&, const std::vector&); - std::int64_t get_output_arg(std::size_t n) const - { - return output_arg < 0 ? n + output_arg : output_arg; - } - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return get_output_arg(shapes.size()); - } - - friend std::ostream& operator<<(std::ostream& os, const code_object_op& op) - { - os << op.name() << "["; - os << "code_object=" << op.code_object.size() << ","; - os << "symbol_name=" << op.symbol_name << ","; - os << "global=" << op.global << ","; - os << "local=" << op.local << ","; - if(op.output_arg != -1) - os << "output_arg=" << op.output_arg << ","; - os << "]"; - return os; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_gen.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_gen.hpp deleted file mode 100644 index 03dad2364..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_gen.hpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILE_GEN_HPP -#define MIGRAPHX_GUARD_GPU_COMPILE_GEN_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct shape; -struct operation; - -namespace gpu { - -struct context; - -namespace gen { - -struct vectorize -{ - std::size_t size = 1; - std::size_t axis = 0; - static vectorize elements(std::size_t axis, const std::vector& inputs); - static vectorize elements(context& ctx, std::size_t axis, const std::vector& inputs); - static vectorize elements(std::size_t axis, - const std::vector& inputs, - const std::vector& sizes); - std::string str() const; -}; -struct preload -{ - std::vector args = {}; - static preload broadcasts(std::size_t axis, const std::vector& inputs); - bool is_preloading() const; - std::string str() const; -}; -struct tile -{ - enum mode - { - store, - load, - none - }; - std::vector args = {}; - std::size_t axis = 0; - std::size_t ntiles = 0; - std::size_t block_size = 0; - std::vector inner{}; - std::vector outer{}; - static tile elements(const std::vector& inputs, std::size_t noutputs); - // bool is_preloading() const; - std::string str() const; -}; - -MIGRAPHX_GPU_EXPORT std::size_t find_fast_axis(const shape& input); -MIGRAPHX_GPU_EXPORT std::size_t find_fast_axis(const std::vector& inputs); - -std::string make_transformer_args(std::vector transformers); - -template -std::string make_transformer_args(Ts... xs) -{ - return make_transformer_args({xs.str()...}); -} - -std::string -generate_pointwise(const module& pm, const std::string& name, bool always_return_tuple = false); - -std::string generate_reduce(module m, const std::string& name); - -std::string generate_name_from_ops(const module& m, const std::string& postname = ""); - -struct reduce_op -{ - std::vector inputs = {}; - std::string reduction = ""; - std::string init = "0"; - std::string read = "op::id{}"; - std::string write = "op::id{}"; - - void set(instruction_ref ins, const operation& op); - void set(const std::string& name, const shape& input, const shape& output); - std::string str() const; - static std::string generate(instruction_ref ins, const std::vector& x); -}; - -} // namespace gen -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_COMPILE_GEN_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hip.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hip.hpp deleted file mode 100644 index d2fa4bcb6..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hip.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_COMPILE_HIP_HPP -#define MIGRAPHX_GUARD_RTGLIB_COMPILE_HIP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -#ifdef MIGRAPHX_USE_HIPRTC -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_HIPRTC); -#endif - -struct hiprtc_src_file -{ - hiprtc_src_file() = default; - hiprtc_src_file(const src_file& s) : path(s.path.string()), content(s.content) {} - std::string path; - std::string content; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.path, "path"), f(self.content, "content")); - } -}; - -MIGRAPHX_GPU_EXPORT bool hip_has_flags(const std::vector& flags); - -MIGRAPHX_GPU_EXPORT std::vector> -compile_hip_src_with_hiprtc(std::vector srcs, - const std::vector& params, - const std::string& arch); - -MIGRAPHX_GPU_EXPORT std::vector> -compile_hip_src(const std::vector& srcs, - const std::vector& params, - const std::string& arch); - -MIGRAPHX_GPU_EXPORT std::string enum_params(std::size_t count, std::string param); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp deleted file mode 100644 index 60b8f20a8..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hip_code_object.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILE_HIP_CODE_OBJECT_HPP -#define MIGRAPHX_GUARD_GPU_COMPILE_HIP_CODE_OBJECT_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_compile_options -{ - std::size_t global; - std::size_t local; - std::vector inputs; - shape output; - std::string kernel_name = "kernel"; - std::vector params = {}; - std::vector virtual_inputs = {}; - std::vector additional_src_files = {}; - std::int64_t output_arg = -1; - - /** - * @brief Set the launch parameters but allow v to override the values - * - * @param v A value class which can have a "global" and/or "local" keys to override the default - * global and local - * @param compute_global A function used to compute the global based on the local - * @param default_local The defaul local to use if its missing from the v parameter - */ - void set_launch_params(const value& v, - const std::function& compute_global, - std::size_t default_local = 1024); - - void - set_launch_params(const value& v, std::size_t default_global, std::size_t default_local = 1024) - { - set_launch_params( - v, [=](auto) { return default_global; }, default_local); - } - - void emplace_param(std::string_view s) { params.emplace_back(s); } -}; - -/// Compute global for n elements, but max out on target-specific upper limit -MIGRAPHX_GPU_EXPORT std::function -compute_global_for(context& ctx, std::size_t n, std::size_t over = 1); - -MIGRAPHX_GPU_EXPORT operation compile_hip_code_object(context& ctx, - const std::string& content, - hip_compile_options options); - -MIGRAPHX_GPU_EXPORT std::size_t -compute_block_size(context& ctx, std::size_t n, std::size_t max_block_size = 1024); - -template -std::string generate_index_ints(const std::vector& v) -{ - return "index_ints<" + to_string_range(v) + ">{}"; -} - -MIGRAPHX_GPU_EXPORT std::string generate_make_shape(const shape& s); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_COMPILE_HIP_CODE_OBJECT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp deleted file mode 100644 index 380fafa44..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_hipblaslt.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILE_HIPBLASLT_HPP -#define MIGRAPHX_GUARD_GPU_COMPILE_HIPBLASLT_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -struct context; -struct operation; - -namespace gpu { - -struct hipblaslt_op -{ - operation op = op::identity{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op")); - } - - std::string name() const { return "gpu::hipblaslt_op"; } - - shape compute_shape(std::vector inputs) const - { - inputs.push_back(inputs.back()); - return op.compute_shape(inputs); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -MIGRAPHX_REGISTER_OP(hipblaslt_op); - -struct compile_hipblaslt -{ - context* ctx = nullptr; - std::string name() const { return "gpu::compile_hipblaslt"; } - void apply(module& m) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_COMPILE_HIPBLASLT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_miopen.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_miopen.hpp deleted file mode 100644 index 03dd669e5..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_miopen.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILE_MIOPEN_HPP -#define MIGRAPHX_GUARD_GPU_COMPILE_MIOPEN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -struct context; -struct operation; - -namespace gpu { - -struct compile_miopen -{ - context* ctx = nullptr; - std::string name() const { return "gpu::compile_miopen"; } - void apply(module& m) const; - std::size_t compile(operation& op, instruction_ref ins) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_COMPILE_MIOPEN_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_ops.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_ops.hpp deleted file mode 100644 index 6986822a5..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_ops.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILE_OPS_HPP -#define MIGRAPHX_GUARD_GPU_COMPILE_OPS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace gpu { - -struct context; - -struct MIGRAPHX_GPU_EXPORT compile_ops -{ - context* ctx = nullptr; - bool exhaustive_tune = false; - std::string name() const { return "gpu::compile_ops"; } - void apply(module& m) const; -}; - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_COMPILE_OPS_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp deleted file mode 100644 index 8e6dc229a..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compile_pointwise.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILE_POINTWISE_HPP -#define MIGRAPHX_GUARD_GPU_COMPILE_POINTWISE_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -operation -compile_pointwise(context& ctx, const std::vector& in_shapes, const_module_ref pm); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_COMPILE_POINTWISE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compiler.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compiler.hpp deleted file mode 100644 index 30f927051..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/compiler.hpp +++ /dev/null @@ -1,201 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_COMPILER_HPP -#define MIGRAPHX_GUARD_GPU_COMPILER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct compiler_replace -{ - compiler_replace() = default; - - compiler_replace(const operation& op) : code_objects{{op}} {} - - template - compiler_replace(const operation& op, F f) : code_objects{{op}}, replace_fn(make_replace(f)) - { - } - - template - compiler_replace(const operation& op, F f, Trace t) - : code_objects{{op}}, replace_fn(make_replace(f)), trace_fn(t) - { - } - - template - compiler_replace(const std::vector& op, F f) - : code_objects{op}, replace_fn(make_replace_all(f)) - { - } - - template - compiler_replace(const std::vector& op, F f, Trace t) - : code_objects{op}, replace_fn(make_replace_all(f)), trace_fn(t) - { - } - - std::vector code_objects = {}; - std::function replace_fn = - nullptr; - std::function trace_fn = nullptr; - - template - static auto make_replace(F f) - { - return [=](const compiler_replace& cr, module& m, instruction_ref ins) { - f(m, ins, cr.code_objects.front()); - }; - } - - template - static auto make_replace_all(F f) - { - return [=](const compiler_replace& cr, module& m, instruction_ref ins) { - f(m, ins, cr.code_objects); - }; - } - - void replace(module& m, instruction_ref ins) const - { - if(replace_fn) - replace_fn(*this, m, ins); - else - { - if(code_objects.size() != 1) - { - MIGRAPHX_THROW("Provide custom replace function to insert multiple code objects\n"); - } - m.replace_instruction(ins, code_objects.front(), ins->inputs()); - } - } - - void trace(std::ostream& os, instruction_ref ins) const - { - if(trace_fn) - trace_fn(os, ins); - } -}; - -using compiler_compile = - std::function; -using compiler_compile_op = - std::function& inputs, const value&)>; -using compiler_tuning_config = - std::function(context&, instruction_ref, const operation&, bool)>; - -MIGRAPHX_GPU_EXPORT void register_compiler(const std::string& name, - compiler_compile c, - compiler_compile_op cop, - compiler_tuning_config ctg); - -MIGRAPHX_GPU_EXPORT bool has_compiler_for(const std::string& name); -MIGRAPHX_GPU_EXPORT compiler_replace compile(context& ctx, - instruction_ref ins, - const operation& op, - const value& solution); -MIGRAPHX_GPU_EXPORT operation compile_op(const std::string& name, - context& ctx, - const std::vector& inputs, - const value& v); -MIGRAPHX_GPU_EXPORT optional -get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool exhaustive); - -template -void register_compiler() -{ - T c; - for(auto&& name : c.names()) - { - register_compiler( - name, - [=](auto&&... xs) { - return c.invoke_compile(rank<1>{}, std::forward(xs)...); - }, - [=](auto&&... xs) { return c.compile_op(std::forward(xs)...); }, - [=](auto&&... xs) { return c.get_tuning_config(std::forward(xs)...); }); - } -} - -struct register_compiler_action -{ - template - static void apply() - { - register_compiler(); - } -}; - -template -using auto_register_compiler = auto_register; - -template -struct compiler : auto_register_compiler -{ - const Derived& derived() const { return static_cast(*this); } - optional - get_tuning_config(context&, instruction_ref, const operation&, bool) const - { - return nullopt; - } - operation compile_op(context&, const std::vector&, const value&) const { return {}; } - - template - auto invoke_compile( - rank<1>, context& ctx, instruction_ref ins, operation op, const value& solution) const - -> decltype(std::declval().compile(ctx, ins, std::move(op), solution)) - { - return derived().compile(ctx, ins, std::move(op), solution); - } - - template - auto invoke_compile( - rank<0>, context& ctx, instruction_ref ins, operation op, const value& solution) const - -> decltype(std::declval().compile(ctx, ins, std::move(op))) - { - assert(solution.empty()); - (void)solution; - return derived().compile(ctx, ins, std::move(op)); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_GPU_COMPILER_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/concat_gpu_opt.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/concat_gpu_opt.hpp deleted file mode 100644 index d5d2f1197..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/concat_gpu_opt.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CONCAT_GPU_OPT_HPP -#define MIGRAPHX_GUARD_RTGLIB_CONCAT_GPU_OPT_HPP - -#include -#include -#include - -namespace migraphx { -namespace gpu { - -struct concat_gpu_optimization -{ - std::string allocate() const { return "hip::allocate"; } - optional get_concat(const migraphx::operation& op) const - { - if(op.name() != "gpu::precompile_op") - return nullopt; - auto r = from_value(op.to_value().at("op")); - if(r.name() == "concat") - return any_cast(r); - return nullopt; - } -}; - -} // namespace gpu - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/config.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/config.hpp deleted file mode 100644 index cd8c6702b..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/config.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef MIGRAPHX_GUARD_GPU_CONFIG_HPP -#define MIGRAPHX_GUARD_GPU_CONFIG_HPP - -#include -#include - -#endif // MIGRAPHX_GUARD_GPU_CONFIG_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/context.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/context.hpp deleted file mode 100644 index 7a1a7d34b..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/context.hpp +++ /dev/null @@ -1,399 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CONTEXT_HPP -#define MIGRAPHX_GUARD_RTGLIB_CONTEXT_HPP - -#include -#include -#include -#if !MIGRAPHX_USE_MIOPEN -#include -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_NULL_STREAM) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_NSTREAMS) - -using hip_event_ptr = MIGRAPHX_MANAGE_PTR(hipEvent_t, hipEventDestroy); - -struct hip_device -{ - hip_device() : device_props{} { add_stream(); } - - hip_device(std::size_t id, std::size_t n) : device_id(id) - { - auto status = hipGetDeviceProperties(&device_props, device_id); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to allocate stream"); - - for(std::size_t i = 0; i < n; i++) - add_stream(); - } - - struct stream - { - using hip_stream_ptr = MIGRAPHX_MANAGE_PTR(hipStream_t, hipStreamDestroy); - - stream() {} - - stream(std::size_t device_number) : id(device_number) {} - - void setup() const { set_device(id); } - - static hip_stream_ptr create_stream() - { - hipStream_t result = nullptr; - auto status = hipStreamCreateWithFlags(&result, hipStreamNonBlocking); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to allocate stream"); - return hip_stream_ptr{result}; - } - - hipStream_t get() - { - if(not enabled(MIGRAPHX_ENABLE_NULL_STREAM{})) - { - setup(); - if(s == nullptr) - s = create_stream(); - assert(s.get() != nullptr); - return s.get(); - } - return nullptr; - } - -#if MIGRAPHX_USE_MIOPEN - auto create_miopen_handle() - { - if(not enabled(MIGRAPHX_ENABLE_NULL_STREAM{})) - return make_obj(&miopenCreateWithStream, get()); - else - return make_obj(&miopenCreate); - } - - auto get_miopen() - { - setup(); - if(mihandle == nullptr) - mihandle = create_miopen_handle(); - assert(mihandle.get() != nullptr); - return mihandle.get(); - } -#endif - -#if MIGRAPHX_USE_ROCBLAS - auto get_rocblas() - { - setup(); - if(rbhandle == nullptr) - rbhandle = create_rocblas_handle_ptr(get()); - assert(rbhandle.get() != nullptr); - return rbhandle.get(); - } -#endif - -#if MIGRAPHX_USE_HIPBLASLT - auto get_hipblaslt() - { - setup(); - if(hblthandle == nullptr) - { - hblthandle = create_hipblaslt_handle_ptr(); - } - assert(hblthandle.get() != nullptr); - return hblthandle.get(); - } -#endif - - void wait() const - { - if(s == nullptr) - return; - setup(); - auto status = hipStreamSynchronize(s.get()); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to wait."); - } - - void wait(hipEvent_t event) - { - setup(); - auto status = hipStreamWaitEvent(get(), event, 0); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to wait."); - } - - void record(hipEvent_t event) - { - setup(); - auto status = hipEventRecord(event, get()); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to record."); - } - - private: - std::size_t id = 0; - shared s = nullptr; -#if MIGRAPHX_USE_MIOPEN - shared mihandle = nullptr; -#endif -#if MIGRAPHX_USE_ROCBLAS - shared rbhandle = nullptr; -#endif - -#if MIGRAPHX_USE_HIPBLASLT - shared hblthandle = nullptr; -#endif - }; - - void add_stream() { streams.emplace_back(device_id); } - - stream& get_stream() { return streams.at(current_stream); } - - stream& get_stream(std::size_t n) { return streams.at(n); } - - const stream& get_stream() const { return streams.at(current_stream); } - - const stream& get_stream(std::size_t n) const { return streams.at(n); } - - void set_stream(std::size_t n) { current_stream = n; } - - std::size_t nstreams() const { return streams.size(); } - - std::size_t stream_id() const { return current_stream; } - - std::string get_device_name() const { return device_props.gcnArchName; } - - std::string get_gfx_name() const { return trim(split_string(get_device_name(), ':').front()); } - - std::size_t get_device_major() const { return device_props.major; } - - std::size_t get_device_minor() const { return device_props.minor; } - - std::size_t get_cu_count() const { return device_props.multiProcessorCount; } - - std::size_t get_max_workitems_per_cu() const - { - return device_props.maxThreadsPerMultiProcessor; - } - - std::size_t get_max_workitems_per_block() const { return device_props.maxThreadsPerBlock; } - - std::size_t get_wavefront_size() const { return device_props.warpSize; } - - private: - std::size_t device_id = 0; - std::size_t current_stream = 0; - std::vector streams; - hipDeviceProp_t device_props; - - public: - std::unordered_map preallocations{}; -}; - -struct context -{ - struct auto_save_problem_cache : problem_cache - { - auto_save_problem_cache() : problem_cache{} {} - - bool auto_save = false; - - auto_save_problem_cache(const auto_save_problem_cache&) = delete; - auto_save_problem_cache& operator=(const auto_save_problem_cache&) = delete; - virtual ~auto_save_problem_cache() - { - if(auto_save) - this->save(); - } - }; - context(std::size_t device_id = 0, std::size_t n = value_of(MIGRAPHX_NSTREAMS{}, 1)) - : current_device(std::make_shared(device_id, n)), - begin_event(create_event()), - finish_event(create_event()), - pc(std::make_shared()) - { - } - - hip_device& get_current_device() - { - assert(current_device != nullptr); - return *current_device; - } - - const hip_device& get_current_device() const - { - assert(current_device != nullptr); - return *current_device; - } - - bool get_exhaustive_tune_flag() const { return exhaustive_tune; } - - void set_exhaustive_tune_flag(bool t) { exhaustive_tune = t; } - - hip_device::stream& get_stream() { return get_current_device().get_stream(); } - hip_device::stream& get_stream(std::size_t n) { return get_current_device().get_stream(n); } - - const hip_device::stream& get_stream() const { return get_current_device().get_stream(); } - const hip_device::stream& get_stream(std::size_t n) const - { - return get_current_device().get_stream(n); - } - - void set_stream(std::size_t n) { get_current_device().set_stream(n); } - - void create_events(std::size_t num_of_events) - { - for(std::size_t i = events.size(); i < num_of_events + 1; ++i) - events.emplace_back(create_event()); - } - - hipEvent_t get_event(std::size_t i) const { return events.at(i).get(); } - - std::vector literals{}; - void finish() const { get_stream().wait(); } - - static hip_event_ptr create_event() - { - hipEvent_t event; - auto status = hipEventCreateWithFlags(&event, hipEventDisableTiming); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to create event"); - return hip_event_ptr{event}; - } - - static hip_event_ptr create_event_for_timing() - { - hipEvent_t event; - auto status = hipEventCreate(&event); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to create event"); - return hip_event_ptr{event}; - } - - value to_value() const - { - value result; - result["events"] = events.size(); - result["streams"] = current_device->nstreams(); - - return result; - } - - void from_value(const value& v) - { - auto v_events = v.at("events"); - std::size_t n_events = v_events.without_key().to(); - this->create_events(n_events - 1); - - auto v_streams = v.at("streams"); - std::size_t n_streams = v_streams.without_key().to(); - - auto device = get_device_id(); - this->current_device = std::make_shared(device, n_streams); - } - - void wait_for(any_ptr queue) - { - auto status = hipEventRecord(begin_event.get(), queue.get()); - if(status != hipSuccess) - MIGRAPHX_THROW("failed to record " + hip_error(status)); - - get_stream().wait(begin_event.get()); - } - - void finish_on(any_ptr queue) - { - get_stream().record(finish_event.get()); - - auto status = hipStreamWaitEvent(queue.get(), finish_event.get(), 0); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to wait on event " + hip_error(status)); - } - - any_ptr get_queue() { return get_stream().get(); } - - std::pair get_perf_events() const - { - if(measure_perf) - return std::make_pair(start_event.get(), stop_event.get()); - return std::make_pair(nullptr, nullptr); - } - - static float get_elapsed_ms(hipEvent_t start, hipEvent_t stop) - { - float result = 0; - if(start != nullptr and stop != nullptr) - { - auto status = hipEventElapsedTime(&result, start, stop); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed hipEventElapsedTime: " + hip_error(status)); - } - return result; - } - - problem_cache& get_problem_cache() { return *pc; } - void load_problem_cache() - { - pc->load(); - pc->auto_save = true; - } - - private: - // TODO: Make this a vector to support multiple devices - std::shared_ptr current_device; - std::vector> events; - bool exhaustive_tune = false; - bool measure_perf = false; - // for event perf timing - shared start_event = nullptr; - shared stop_event = nullptr; - // for stream syncronization - shared begin_event = nullptr; - shared finish_event = nullptr; - std::shared_ptr pc = nullptr; -}; - -inline void migraphx_to_value(value& v, const context& ctx) { v = ctx.to_value(); } -inline void migraphx_from_value(const value& v, context& ctx) { ctx.from_value(v); } - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/contiguous.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/contiguous.hpp deleted file mode 100644 index 638f4571a..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/contiguous.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CONTIGUOUS_HPP -#define MIGRAPHX_GUARD_RTGLIB_CONTIGUOUS_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct miopen_contiguous : unary_device -{ - std::string name() const { return "gpu::contiguous"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(2); - auto lens = inputs.at(0).lens(); - auto t = inputs.at(0).type(); - return {t, lens}; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/convolution.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/convolution.hpp deleted file mode 100644 index 1a6d1bc24..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/convolution.hpp +++ /dev/null @@ -1,352 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_CONVOLUTION_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_CONVOLUTION_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -inline shape reshape_if_1d(const shape& input) -{ - shape new_shape{input}; - auto dims = new_shape.lens(); - - if(dims.size() == 3) - { - std::vector new_dims = dims; - new_dims.insert(new_dims.begin() + 2, 1); - new_shape = shape{input.type(), new_dims}; - } - return new_shape; -} -#if MIGRAPHX_USE_MIOPEN -template -struct miopen_convolution -{ - Op op; - shared cd = nullptr; - miopenConvFwdAlgorithm_t algo{}; -#ifdef MIGRAPHX_HAS_FIND_2_API - value::binary solution_object{}; - shared solution_ptr = nullptr; -#endif - uint64_t solution_id = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op"), -#ifdef MIGRAPHX_HAS_FIND_2_API - f(self.solution_object, "solution_object"), -#endif - f(self.algo, "algo"), - f(self.solution_id, "solution_id")); - } - - std::string name() const { return "gpu::" + op.name(); } - - inline shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, op}.has(4); - std::vector conv_inputs(inputs.begin(), inputs.begin() + 2); - check_shapes{conv_inputs, *this} - .max_ndims(5) - .packed_layouts({{0, 1, 2}, {0, 1, 2, 3}, {0, 2, 3, 1}, {0, 1, 2, 3, 4}}) - .same_layout(); - return migraphx::compute_shape(op, conv_inputs); - } - - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - auto x_desc = make_tensor(reshape_if_1d(args[0].get_shape())); - auto w_desc = make_tensor(reshape_if_1d(args[1].get_shape())); - auto y_desc = make_tensor(reshape_if_1d(output_shape)); - auto* miopen_stream_handle = ctx.get_stream().get_miopen(); - auto workspace_size = args[2].get_shape().bytes(); - -#ifdef MIGRAPHX_HAS_FIND_2_API - { - const miopenTensorArgument_t tensor_args[3] = { - {miopenTensorConvolutionX, nullptr, args[0].implicit()}, - {miopenTensorConvolutionW, nullptr, args[1].implicit()}, - {miopenTensorConvolutionY, nullptr, args[3].implicit()}, - }; - - if(solution_ptr.get() == nullptr) - MIGRAPHX_THROW("MIOpen " + op.name() + " : Load MIOpen Solution before running it"); - - auto status = miopenRunSolution(miopen_stream_handle, - solution_ptr.get(), - 3, - tensor_args, - args[2].implicit(), - workspace_size); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen " + op.name() + - " : running convolution using find_2.0 failed"); - - return args[3]; - } -#else - // else use immediate mode - if(solution_id == 0) - MIGRAPHX_THROW("MIOpen " + op.name() + " : invalid solution ID"); - - auto status = miopenConvolutionForwardImmediate(miopen_stream_handle, - w_desc.get(), - args[1].implicit(), - x_desc.get(), - args[0].implicit(), - cd.get(), - y_desc.get(), - args[3].implicit(), - args[2].implicit(), - workspace_size, - solution_id); - - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen " + op.name() + ": running convolution failed"); - return args[3]; -#endif - } - - void set_conv_descriptor() - { - cd = - (op.name() == "convolution_backwards") ? make_convolution_backwards(op) : make_conv(op); - } - - value compile(migraphx::context& ctx, const shape& output, const std::vector& input) - { - set_conv_descriptor(); - auto ws = find(any_cast(ctx), output, input); - return {{"workspace", ws.bytes()}}; - } - - shape find(context& ctx, const shape& output_shape, const std::vector& inputs) - { - shape workspace_shape{}; - auto x_desc = make_tensor(reshape_if_1d(inputs[0])); - auto w_desc = make_tensor(reshape_if_1d(inputs[1])); - auto y_desc = make_tensor(reshape_if_1d(output_shape)); - - auto* miopen_stream_handle = ctx.get_stream().get_miopen(); - std::size_t workspace_size = 0; - auto status = miopenConvolutionForwardGetWorkSpaceSize(miopen_stream_handle, - w_desc.get(), - x_desc.get(), - cd.get(), - y_desc.get(), - &workspace_size); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen" + op.name() + " : Failed to get forward workspace size"); - - workspace_shape = shape{shape::int8_type, {workspace_size}}; - - const auto& x_shape = inputs[0]; - const auto& w_shape = inputs[1]; - - unsigned long seed = 0; -#ifdef MIGRAPHX_HAS_FIND_2_API - { - auto conv_problem = make_obj( - &miopenCreateConvProblem, cd.get(), miopenProblemDirectionForward); - - set_tensor_descriptor(miopenTensorConvolutionX, x_desc, conv_problem); - set_tensor_descriptor(miopenTensorConvolutionW, w_desc, conv_problem); - bool preallocate = false; -#ifdef MIGRAPHX_PREALLOCATE_MIOPEN_BUFFERS - // MIOpen has APIs to pass pre-allocated buffers starting from rocm-5.6 - preallocate = true; -#endif - auto x = preallocate ? to_gpu(generate_argument(x_shape, seed++, random_mode::random)) - : argument{inputs[0]}; - auto w = preallocate ? to_gpu(generate_argument(w_shape, seed++, random_mode::random)) - : argument{inputs[1]}; - auto y = preallocate ? allocate_gpu(output_shape) : argument{inputs[2]}; - auto workspace = - preallocate ? allocate_gpu(workspace_shape) : migraphx::argument(workspace_shape); - - set_tensor_descriptor(miopenTensorConvolutionY, y_desc, conv_problem); - - const miopenTensorArgument_t tensor_args[3] = { - {miopenTensorConvolutionX, nullptr, x.implicit()}, - {miopenTensorConvolutionW, nullptr, w.implicit()}, - {miopenTensorConvolutionY, nullptr, y.implicit()}, - }; - - solution_ptr = find_solution(miopen_stream_handle, - 3, - tensor_args, - workspace.implicit(), - workspace_size, - conv_problem.get(), - ctx.get_exhaustive_tune_flag()); - - status = miopenGetSolutionWorkspaceSize(solution_ptr.get(), &workspace_size); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen" + op.name() + " : failed to get solution's workspace size"); - - std::size_t solution_size; - status = miopenGetSolutionSize(solution_ptr.get(), &solution_size); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen" + op.name() + ": Failed to fetch solution size"); - - auto solution_binary = std::vector{}; - solution_binary.resize(solution_size); - - status = miopenSaveSolution(solution_ptr.get(), solution_binary.data()); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen" + op.name() + ": Saving solution failed"); - solution_object = value::binary{solution_binary.data(), solution_size}; - return shape{shape::int8_type, {workspace_size}}; - } -#else - auto x = to_gpu(generate_argument(x_shape, seed++, random_mode::random)); - auto w = to_gpu(generate_argument(w_shape, seed++, random_mode::random)); - auto y = allocate_gpu(output_shape); - auto workspace = allocate_gpu(workspace_shape); - int algo_count = 1; - miopenConvAlgoPerf_t perf; - status = miopenFindConvolutionForwardAlgorithm(ctx.get_stream().get_miopen(), - x_desc.get(), - x.implicit(), - w_desc.get(), - w.implicit(), - cd.get(), - y_desc.get(), - y.implicit(), - 1, - &algo_count, - &perf, - workspace.implicit(), - workspace_size, - ctx.get_exhaustive_tune_flag()); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen " + op.name() + " : find convolution failed"); - algo = perf.fwd_algo; - size_t solution_count; - - status = miopenConvolutionForwardGetSolutionCount(ctx.get_stream().get_miopen(), - w_desc.get(), - x_desc.get(), - cd.get(), - y_desc.get(), - &solution_count); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen " + op.name() + ": get solution count failed"); - - std::vector solutions(solution_count); - - status = miopenConvolutionForwardGetSolution(ctx.get_stream().get_miopen(), - w_desc.get(), - x_desc.get(), - cd.get(), - y_desc.get(), - solution_count, - &solution_count, - solutions.data()); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen " + op.name() + ": get solution failed"); - - solution_id = solutions.front().solution_id; - - return shape{shape::int8_type, {perf.memory}}; -#endif - } - - void finalize(context& ctx, const shape& output_shape, const std::vector& inputs) - { -#ifdef MIGRAPHX_HAS_FIND_2_API - { - (void)(ctx); // avoid warnings - (void)(output_shape); - (void)(inputs); - // load solution - if(solution_ptr == nullptr) - { - miopenSolution_t ptr; - auto status = - miopenLoadSolution(&ptr, - reinterpret_cast(solution_object.data()), - solution_object.size()); - solution_ptr = miopen_solution{ptr}; - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen " + op.name() + ": loading convolution solution failed"); - } - } -#else - // Use immediate mode API - { - set_conv_descriptor(); - if(solution_id == 0) - { - // Check that workspace hasn't changed - auto size = inputs.at(2).bytes(); - auto ws = find(ctx, output_shape, inputs); - if(ws.bytes() > size) - MIGRAPHX_THROW("MIOpen " + op.name() + - ": workspace has changed during finalization."); - } - - auto x_desc = make_tensor(reshape_if_1d(inputs[0])); - auto w_desc = make_tensor(reshape_if_1d(inputs[1])); - auto y_desc = make_tensor(reshape_if_1d(output_shape)); - - auto status = miopenConvolutionForwardCompileSolution(ctx.get_stream().get_miopen(), - w_desc.get(), - x_desc.get(), - cd.get(), - y_desc.get(), - solution_id); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen Convolution: compile solution failed"); - } -#endif - } - - inline std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/arg_op.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/arg_op.hpp deleted file mode 100644 index db8505b09..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/arg_op.hpp +++ /dev/null @@ -1,172 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_ARG_OP_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_ARG_OP_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -template -struct val_index -{ - T val; - int64_t index; -}; - -template -MIGRAPHX_DEVICE_CONSTEXPR val_index make_val_index(T v) -{ - return {v, -1}; -} - -template -MIGRAPHX_DEVICE_CONSTEXPR val_index make_val_index(T v, int64_t i) -{ - return {v, i}; -} - -struct argmax_op_first_index -{ - template - MIGRAPHX_DEVICE_CONSTEXPR val_index operator()(val_index x, val_index y) const - { - if(x.val > y.val) - return x; - else if(x.val < y.val) - return y; - else - { - return (x.index < y.index) ? x : y; - } - } - - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return lowest(); } -}; - -struct argmax_op_last_index -{ - template - MIGRAPHX_DEVICE_CONSTEXPR val_index operator()(val_index x, val_index y) const - { - if(x.val > y.val) - return x; - else if(x.val < y.val) - return y; - else - { - return (x.index > y.index) ? x : y; - } - } - - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return lowest(); } -}; - -struct argmin_op_first_index -{ - template - MIGRAPHX_DEVICE_CONSTEXPR val_index operator()(val_index x, val_index y) const - { - if(x.val < y.val) - return x; - else if(x.val > y.val) - return y; - else - { - return (x.index < y.index) ? x : y; - } - } - - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return highest(); } -}; - -struct argmin_op_last_index -{ - template - MIGRAPHX_DEVICE_CONSTEXPR val_index operator()(val_index x, val_index y) const - { - if(x.val < y.val) - return x; - else if(x.val > y.val) - return y; - else - { - return (x.index > y.index) ? x : y; - } - } - - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return highest(); } -}; - -template -void arg_op(Op op, hipStream_t stream, const argument& result, const argument& arg, int64_t axis) -{ - auto arg_shape = arg.get_shape(); - auto batch_lens = arg_shape.lens(); - size_t batch_item_num = batch_lens[axis]; - batch_lens[axis] = 1; - migraphx::shape batch_shape{arg_shape.type(), batch_lens}; - migraphx::shape std_arg_shape{arg_shape.type(), arg_shape.lens()}; - - hip_visit_all(arg, std_arg_shape, batch_shape)([&](auto input, auto arg_s, auto batch_s) { - auto* output = device_cast(result.get().data()); - using type = device_type>; - // use one block for items in one batch. - const size_t max_block_size = 256; - const std::size_t block_size = compute_block_size(batch_item_num, max_block_size); - gs_launch(stream, - batch_shape.elements() * block_size, - block_size)([=](auto i, auto idx) __device__ { - auto batch_idx = batch_s.multi(i / block_size); - auto data_idx = batch_idx; - auto init = make_val_index(op.init()); - - auto op_output = - block_reduce(idx, op, init, batch_item_num, [&](auto j) __device__ { - data_idx[axis] = j; - return make_val_index(input[arg_s.index(data_idx)], j); - }); - - if(idx.local == 0) - { - output[batch_s.index(batch_idx)] = op_output.index; - } - }); - }); -} - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/argmax.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/argmax.hpp deleted file mode 100644 index be6023737..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/argmax.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_ARGMAX_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_ARGMAX_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT argmax(hipStream_t stream, - const argument& result, - const argument& arg, - int64_t axis, - bool select_last_index); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/argmin.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/argmin.hpp deleted file mode 100644 index c205fcf72..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/argmin.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_ARGMIN_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_ARGMIN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT argmin(hipStream_t stream, - const argument& result, - const argument& arg, - int64_t axis, - bool select_last_index); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/config.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/config.hpp deleted file mode 100644 index 014a5f3a3..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/config.hpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_CONFIG_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_CONFIG_HPP - -#include -#include - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/contiguous.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/contiguous.hpp deleted file mode 100644 index 5012955de..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/contiguous.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_KERNELS_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_KERNELS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT contiguous(hipStream_t stream, - const argument& result, - const argument& arg); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/fill.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/fill.hpp deleted file mode 100644 index 643b26b2e..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/fill.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_FILL_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_FILL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT fill(hipStream_t stream, const argument& result, unsigned long val); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/logsoftmax.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/logsoftmax.hpp deleted file mode 100644 index 0f08b84b5..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/logsoftmax.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_LOGSOFTMAX_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_LOGSOFTMAX_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT logsoftmax(hipStream_t stream, - const argument& result, - const argument& arg, - int64_t axis); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/multinomial.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/multinomial.hpp deleted file mode 100644 index 7998945b0..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/multinomial.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_MULTINOMIAL_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_MULTINOMIAL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT multinomial(hipStream_t stream, - const argument& result, - const argument& arg0, - const argument& arg1); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/nonzero.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/nonzero.hpp deleted file mode 100644 index a470a337a..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/nonzero.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_NONZERO_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_NONZERO_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -argument MIGRAPHX_DEVICE_EXPORT nonzero(hipStream_t stream, - const argument& result, - const argument& arg_data); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/prefix_scan_sum.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/prefix_scan_sum.hpp deleted file mode 100644 index a51815ec4..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/prefix_scan_sum.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_DEVICE_PREFIX_SCAN_SUM_HPP -#define MIGRAPHX_GUARD_DEVICE_PREFIX_SCAN_SUM_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT prefix_scan_sum(hipStream_t stream, - const argument& result, - const argument& arg, - int32_t axis, - bool exclusive, - bool reverse); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_DEVICE_PREFIX_SCAN_SUM_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/reverse.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/reverse.hpp deleted file mode 100644 index 1414314e6..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/reverse.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_REVERSE_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_REVERSE_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -argument MIGRAPHX_DEVICE_EXPORT reverse(hipStream_t stream, - argument result, - argument arg1, - const std::vector& axes); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/rnn_variable_seq_lens.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/rnn_variable_seq_lens.hpp deleted file mode 100644 index 950848057..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/rnn_variable_seq_lens.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_RNN_VARIABLE_SEQ_LENS_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_RNN_VARIABLE_SEQ_LENS_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -void MIGRAPHX_DEVICE_EXPORT rnn_var_sl_shift_sequence(hipStream_t stream, - const argument& result, - const argument& arg_hs, - const argument& arg_sl); - -void MIGRAPHX_DEVICE_EXPORT rnn_var_sl_shift_output(hipStream_t stream, - const argument& result, - const argument& arg_hs, - const argument& arg_sl, - bool is_reverse); - -void MIGRAPHX_DEVICE_EXPORT rnn_var_sl_last_output(hipStream_t stream, - const argument& result, - const argument& arg_hs, - const argument& arg_sl, - bool is_reverse); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/topk.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/topk.hpp deleted file mode 100644 index b1fb4e8e2..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device/topk.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_DEVICE_TOPK_HPP -#define MIGRAPHX_GUARD_RTGLIB_DEVICE_TOPK_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -namespace device { - -argument MIGRAPHX_DEVICE_EXPORT topk_smallest(hipStream_t stream, - const argument& val_res, - const argument& ind_res, - const argument& arg, - int64_t k, - int64_t axis); - -argument MIGRAPHX_DEVICE_EXPORT topk_largest(hipStream_t stream, - const argument& val_res, - const argument& ind_res, - const argument& arg, - int64_t k, - int64_t axis); - -} // namespace device -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device_name.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device_name.hpp deleted file mode 100644 index bdd9530aa..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/device_name.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_DEVICE_NAME_HPP -#define MIGRAPHX_GUARD_GPU_DEVICE_NAME_HPP - -#include -#include - -struct hipDeviceProp_t; - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_GPU_EXPORT std::string get_device_name(); - -MIGRAPHX_GPU_EXPORT int get_device_id(); - -MIGRAPHX_GPU_EXPORT bool gfx_has_fp8fnuz_intrinsics(); - -MIGRAPHX_GPU_EXPORT bool gfx_has_fp8ocp_intrinsics(); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_DEVICE_NAME_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_ck.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_ck.hpp deleted file mode 100644 index ee726b5b7..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_ck.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_FUSE_CK_HPP -#define MIGRAPHX_GUARD_GPU_FUSE_CK_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -namespace gpu { - -struct fuse_ck -{ - context* ctx = nullptr; - std::string name() const { return "gpu::fuse_ck"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_FUSE_CK_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_mlir.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_mlir.hpp deleted file mode 100644 index e1cb8f0bb..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_mlir.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_FUSE_MLIR_HPP -#define MIGRAPHX_GUARD_GPU_FUSE_MLIR_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -namespace gpu { - -MIGRAPHX_GPU_EXPORT bool mlir_enabled(); - -struct MIGRAPHX_GPU_EXPORT fuse_mlir -{ - context* ctx = nullptr; - bool enable_extra = false; - std::string name() const { return "gpu::fuse_mlir"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_FUSE_MLIR_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_ops.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_ops.hpp deleted file mode 100644 index fc8ef2256..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/fuse_ops.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_FUSE_OPS_HPP -#define MIGRAPHX_GUARD_RTGLIB_FUSE_OPS_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace gpu { - -struct MIGRAPHX_GPU_EXPORT fuse_ops -{ - context* ctx = nullptr; - bool fast_math = true; - std::string name() const { return "gpu::fuse_ops"; } - void apply(module& m) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm.hpp deleted file mode 100644 index 23f053dd5..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm.hpp +++ /dev/null @@ -1,163 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_GEMM_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_GEMM_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; -shape transpose_batch(const shape& s, unsigned trans_batch); -void blas_shape(const shape& s); - -template -struct rocblas_gemm -{ - Op op; - float alpha = 1; - float beta = 0; - bool compute_fp32 = false; - unsigned trans_batch = 0; - int32_t solution_idx = 0; - template - static auto reflect(Self& self, F f) - { - return pack_join(migraphx::reflect(self.op, f), - pack(f(self.alpha, "alpha"), - f(self.beta, "beta"), - f(self.compute_fp32, "compute_fp32"), - f(self.trans_batch, "trans_batch"), - f(self.solution_idx, "solution_idx"))); - } - - std::string name() const - { - if(contains(op.name(), "quant_")) - { - return "gpu::quant_gemm"; - } - return "gpu::gemm"; - } - - shape compute_shape(const std::vector& inputs) const - { - std::vector in_shapes(inputs); - in_shapes.pop_back(); - // When input shapes are A, B, C the GEMM equation is C  =  α AB+ β C where α, β are - // scalars - check_shapes{in_shapes, *this}.has(2, 3); - blas_shape(inputs[0]); - blas_shape(inputs[1]); - // if gemm and add are fused - if(in_shapes.size() > 2) - { - auto cmat_shape = in_shapes.back(); - check_shapes{{cmat_shape}, *this}.not_transposed().not_broadcasted(); - in_shapes.pop_back(); - blas_shape(cmat_shape); - auto op_out_shape = op.compute_shape(in_shapes); - if(cmat_shape.lens() != op_out_shape.lens()) - { - MIGRAPHX_THROW(this->name() + " : dimension mismatch, operand C: {" + - to_string_range(cmat_shape.lens()) + - "}, cannot add to operand A * B: {" + - to_string_range(op_out_shape.lens()) + "}"); - } - if(cmat_shape.type() != op_out_shape.type()) - { - MIGRAPHX_THROW(this->name() + " : operand C type mismatch, operand C is of type: " + - to_string(cmat_shape.type()) + - ", it must be: " + to_string(op_out_shape.type())); - } - return transpose_batch(op_out_shape, trans_batch); - } - - return transpose_batch(op.compute_shape(in_shapes), trans_batch); - } - - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - if(this->name() == "gpu::gemm" or output_shape.type() == migraphx::shape::float_type) - { - gemm_compute(ctx, output_shape, args, alpha, beta, compute_fp32, solution_idx); - } - else - { - gemm_compute( - ctx, output_shape, args, int32_t(alpha), int32_t(beta), compute_fp32, solution_idx); - } - return args.back(); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } - - void finalize(context& ctx, const shape& output_shape, const std::vector& input_shapes) - { -#ifdef MIGRAPHX_USE_ROCBLAS_TUNING_API - if(solution_idx == 0) - solution_idx = gemm_default_solution(ctx, output_shape, input_shapes); - if(enabled(MIGRAPHX_ENABLE_GEMM_TUNING{}) or ctx.get_exhaustive_tune_flag()) - { - if(this->name() == "gpu::gemm") - { - solution_idx = gemm_finalize( - ctx, output_shape, input_shapes, alpha, beta, compute_fp32, solution_idx); - } - else - { - solution_idx = gemm_finalize(ctx, - output_shape, - input_shapes, - int32_t(alpha), - int32_t(beta), - compute_fp32, - solution_idx); - } - } -#else - // suppress compiler warnings - (void)ctx, (void)output_shape, (void)input_shapes; -#endif // MIGRAPHX_USE_ROCBLAS_TUNING_API - } -}; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_RTGLIB_GPU_GEMM_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm_impl.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm_impl.hpp deleted file mode 100644 index 76891cbb8..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm_impl.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GEMM_IMPL_HPP -#define MIGRAPHX_GUARD_RTGLIB_GEMM_IMPL_HPP - -#include -#include -#include -#include - -// Set this environment variable to "true" to perform GEMM tuning even when the -// --exhaustive-tune option isn't set. Can be used to skip slow convolution tuning. -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_GEMM_TUNING); - -using milliseconds = std::chrono::duration; -using microseconds = std::chrono::duration; - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -/** - * @brief Templated implementations of the compute() and finalize() methods of the Gemm operator. - * For each function there are overloads using either float or int32_t for the arguments - * alpha and beta. - * - * @param ctx . - * @param output_shape . - * @param args . - * @param alpha . - * @param beta . - * @param compute_fp32 . - */ -void gemm_compute(context& ctx, - const shape& output_shape, - const std::vector& args, - float alpha, - float beta, - bool compute_fp32, - int32_t solution_idx); - -void gemm_compute(context& ctx, - const shape& output_shape, - const std::vector& args, - int32_t alpha, - int32_t beta, - bool compute_fp32, - int32_t solution_idx); - -int32_t gemm_finalize(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - float alpha, - float beta, - bool compute_fp32); - -int32_t gemm_finalize(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - int32_t alpha, - int32_t beta, - bool compute_fp32, - int32_t solution_idx); - -int32_t gemm_default_solution(context& ctx, - const shape& output_shape, - const std::vector& input_shapes); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm_softmax_gemm.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm_softmax_gemm.hpp deleted file mode 100644 index 6a63bde37..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/gemm_softmax_gemm.hpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_GEMM_SOFTMAX_GEMM_HPP -#define MIGRAPHX_GUARD_GPU_GEMM_SOFTMAX_GEMM_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct gemm_softmax_gemm -{ - operation op = make_op("dot"); - float scale = 1.0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op"), f(self.scale, "scale")); - } - - std::string name() const { return "gpu::gemm_softmax_gemm"; } - - void check_gemm_shape(const shape& s) const - { - if(not contains(range(s.strides().rbegin(), s.strides().rbegin() + 3), 1) and - not s.scalar()) - MIGRAPHX_THROW("Invalid shape for " + name()); - } - - shape compute_shape(std::vector inputs, const std::vector&) const - { - check_shapes{inputs, *this}.same_ndims(); - if(inputs.size() < 3) - MIGRAPHX_THROW(name() + ": Expected 3 inputs but got " + to_string(inputs.size())); - - const bool is_bias_enabled = inputs.size() == 4; - const bool is_mul_where = inputs.size() == 5; - auto a = inputs[0]; - auto b = inputs[1]; - auto b1 = inputs.back(); - - for(const auto& input : inputs) - { - check_gemm_shape(input); - } - auto gemm0_shape = op.compute_shape({a, b}); - if(is_mul_where) - { - auto select_cond = inputs[2]; - auto select_const = inputs[3]; - if(select_cond.lens() != select_const.lens()) - { - std::stringstream err_msg; - err_msg << name() << ": has inconsistent where op condition and constant size: " - << select_cond << "!=" << select_const; - MIGRAPHX_THROW(err_msg.str()); - } - if(select_cond.lens() != gemm0_shape.lens()) - { - std::stringstream err_msg; - err_msg << name() << ": has inconsistent where op condition size" - << ". Expected: " << gemm0_shape << ". Given: " << select_cond; - MIGRAPHX_THROW(err_msg.str()); - } - } - if(is_bias_enabled) - { - auto bias_shape = inputs[2]; - if(bias_shape.lens() != gemm0_shape.lens()) - { - std::stringstream err_msg; - err_msg << name() << ": has inconsistent bias size" - << ". Expected: " << gemm0_shape << ". Given: " << bias_shape; - MIGRAPHX_THROW(err_msg.str()); - } - } - - return op.compute_shape({gemm0_shape, b1}); - } - - static bool is_ck_supported_type(shape::type_t t) { return contains({shape::half_type}, t); } - static bool is_mlir_supported_type(shape::type_t t) - { - return contains({shape::type_t::float_type, shape::half_type}, t); - } -}; - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_GEMM_SOFTMAX_GEMM_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/group_query_attention.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/group_query_attention.hpp deleted file mode 100644 index b7690c4bf..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/group_query_attention.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_GROUP_QUERY_ATTENTION_HPP -#define MIGRAPHX_GUARD_GPU_GROUP_QUERY_ATTENTION_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct gqa_parameters -{ - float scale; - std::uint32_t batch_size; // Batch size used by input - std::uint32_t sequence_length; // Sequence length used by input - std::uint32_t hidden_size; // Hidden size used by input - std::uint32_t head_size; // Head size - std::uint32_t rotary_embedding_dim; // Rotary embedding dimension. - std::uint32_t num_heads; // num_heads = hidden_size / head_size - std::uint32_t max_sequence_length; // Sequence length used by cos/sin cache - std::uint32_t head_stride; // Head stride - std::uint32_t seq_stride; // Sequence stride - std::uint32_t batch_stride; // Batch stride - std::uint32_t position_ids_format; // Format of position ids - 0 is (1), 1 is (batch_size, - // sequence_length) - std::uint32_t seqlen_present_kv_cache; // Sequence length of present kv-cache (4096 when using - // shared buffer) - bool do_rotary; // Whether to use rotary position embedding. Default value is 0. - std::uint32_t kv_num_heads; // Number of attention heads for k and v - int local_window_size; // left_window_size for local attention. Default value is -1 meaning - // unused. - bool rotary_interleaved; // Rotate using interleaved pattern. Default value is 0 (False). - bool past_present_share_buffer; // Whether to use same buffer for KV-cache inputs and outputs - - std::string make_init_str() const - { - return "MIGRAPHX_MAKE_CONSTANT(float{" + std::to_string(scale) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(batch_size) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(sequence_length) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(hidden_size) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(head_size) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(rotary_embedding_dim) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(num_heads) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(max_sequence_length) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(head_stride) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(seq_stride) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(batch_stride) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(position_ids_format) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(seqlen_present_kv_cache) + - "}), " + "MIGRAPHX_MAKE_CONSTANT(bool{" + - std::to_string(static_cast(do_rotary)) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(uint32_t{" + std::to_string(kv_num_heads) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(int32_t{" + std::to_string(local_window_size) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(bool{" + - std::to_string(static_cast(rotary_interleaved)) + "}), " + - "MIGRAPHX_MAKE_CONSTANT(bool{" + - std::to_string(static_cast(past_present_share_buffer)) + "})"; - } -}; - -static inline gqa_parameters init_params(const std::vector& inputs, const value& v) -{ - auto num_heads = v.at("num_heads").to(); - auto kv_num_heads = v.at("kv_num_heads").to(); - auto do_rotary = v.at("do_rotary").to(); - auto local_window_size = v.at("local_window_size").to(); - auto rotary_interleaved = v.at("rotary_interleaved").to(); - auto scale = v.at("scale").to(); - auto present_kv_seqlen = inputs[1].lens().size() == 4 ? inputs[1].lens()[2] : 0; - - const auto& q_shape = inputs[0]; - auto q_lens = q_shape.lens(); - const std::size_t batch_size = q_lens[0]; - const std::size_t sequence_length = q_lens[2]; - std::size_t head_size = q_lens[3]; - auto q_hidden_size = kv_num_heads * head_size; - - std::size_t rotary_dim = inputs[3].lens()[1] * 2; - auto seq_stride = head_size; - auto head_stride = sequence_length * seq_stride; - auto batch_stride = (num_heads + 2 * kv_num_heads) * head_stride; - auto position_ids_format = sequence_length == 1 ? 1 : 0; - bool past_present_share_buffer = true; - gqa_parameters gqa_params; - gqa_params.batch_size = batch_size; - gqa_params.sequence_length = sequence_length; - gqa_params.hidden_size = q_hidden_size; - gqa_params.head_size = head_size; - gqa_params.rotary_embedding_dim = rotary_dim; - gqa_params.num_heads = num_heads; - gqa_params.max_sequence_length = sequence_length; - gqa_params.seq_stride = head_size; - gqa_params.head_stride = head_stride; - gqa_params.batch_stride = batch_stride; - gqa_params.position_ids_format = position_ids_format; - gqa_params.seqlen_present_kv_cache = present_kv_seqlen; - gqa_params.do_rotary = do_rotary; - gqa_params.kv_num_heads = kv_num_heads; - gqa_params.local_window_size = local_window_size; - gqa_params.rotary_interleaved = rotary_interleaved; - gqa_params.scale = scale; - gqa_params.past_present_share_buffer = past_present_share_buffer; - - return gqa_params; -} - -} // namespace gpu - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_GROUP_QUERY_ATTENTION_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip.hpp deleted file mode 100644 index acd7525d6..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip.hpp +++ /dev/null @@ -1,288 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_HIP_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_HIP_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -MIGRAPHX_GPU_EXPORT std::string hip_error(int error); - -MIGRAPHX_GPU_EXPORT argument allocate_gpu(const shape& s, bool host = false); - -MIGRAPHX_GPU_EXPORT argument register_on_gpu(const argument& arg); - -MIGRAPHX_GPU_EXPORT argument to_gpu(const argument& arg, bool host = false); - -MIGRAPHX_GPU_EXPORT argument from_gpu(const argument& arg); - -MIGRAPHX_GPU_EXPORT void set_device(std::size_t id); - -MIGRAPHX_GPU_EXPORT void gpu_sync(); -MIGRAPHX_GPU_EXPORT void gpu_sync(const context& ctx); - -MIGRAPHX_GPU_EXPORT void gpu_copy(context& ctx, const argument& src, const argument& dst); -MIGRAPHX_GPU_EXPORT void copy_to_gpu(context& ctx, const argument& src, const argument& dst); -MIGRAPHX_GPU_EXPORT void copy_from_gpu(context& ctx, const argument& src, const argument& dst); - -MIGRAPHX_GPU_EXPORT argument get_preallocation(context& ctx, const std::string& id); - -MIGRAPHX_GPU_EXPORT void gpu_fill(context& ctx, const argument& dst, int value = 0); - -struct hip_allocate -{ - shape s; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape")); - } - - std::string name() const { return "hip::allocate"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return s; - } - argument compute(context&, const shape& output_shape, const std::vector&) const - { - return allocate_gpu(output_shape); - } -}; - -struct hip_fill -{ - int value = 0; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.value, "value")); - } - - std::string name() const { return "hip::fill"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(1); - return inputs.front(); - } - argument compute(context& ctx, const shape&, const std::vector& args) const - { - gpu_fill(ctx, args.front(), value); - return args.front(); - } - std::ptrdiff_t output_alias(const std::vector&) const { return 0; } -}; - -struct hip_sync_stream -{ - - std::string name() const { return "hip::sync_stream"; } - shape compute_shape(const std::vector& inputs) const - { - if(inputs.empty()) - return {}; - return inputs.front(); - } - - argument compute(const context& ctx, const shape&, const std::vector& args) const - { - gpu_sync(ctx); - if(args.empty()) - return {}; - return args.front(); - } - - std::ptrdiff_t output_alias(const std::vector& args) const - { - if(args.empty()) - return -1; - return 0; - } -}; - -struct hip_copy_to_gpu -{ - std::string name() const { return "hip::copy_to_gpu"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1, 2).same_type(); - return inputs.at(0); - } - argument compute(context& ctx, const shape&, const std::vector& args) const - { - auto input = register_on_gpu(args[0]); - if(args.size() == 1) - return input; - argument result = args[1].share(); - if(result.get_shape().dynamic()) - { - result = result.reshape(args[0].get_shape()); - } - gpu_copy(ctx, input, result); - // Associate the input since it was registered with hip - return {result.get_shape(), [input, result]() mutable { return result.data(); }}; - } - std::ptrdiff_t output_alias(const std::vector& args) const - { - if(args.size() == 1) - return -1; - return 1; - } -}; - -struct hip_copy_from_gpu -{ - std::string name() const { return "hip::copy_from_gpu"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this, true}.has(1, 2).same_type(); - return inputs.at(0); - } - argument - compute(context& ctx, const dyn_output& dyn_out, const std::vector& args) const - { - if(args.size() == 1) - { - argument result = allocate_gpu(dyn_out.computed_shape, true); - gpu_copy(ctx, args[0], result); - return result; - } - argument input = args[0].share(); - if(input.get_shape().dynamic()) - { - input = input.reshape(args[1].get_shape()); - } - copy_from_gpu(ctx, input, args[1]); - return args[1]; - } - std::ptrdiff_t output_alias(const std::vector& args) const - { - if(args.size() == 1) - return -1; - return 1; - } -}; - -struct hip_copy -{ - std::string name() const { return "hip::copy"; } - shape compute_shape(std::vector inputs) const - { - check_shapes{inputs, *this}.has(2).same_type(); - return inputs.at(1); - } - argument compute(context& ctx, const shape&, std::vector args) const - { - gpu_copy(ctx, args[0], args[1]); - return args[1]; - } - std::ptrdiff_t output_alias(const std::vector&) const { return 1; } -}; - -MIGRAPHX_GPU_EXPORT void -store_preallocated_param(context& ctx, const std::string& id, const argument& a); - -struct hip_allocate_memory -{ - shape s; - std::string id{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.s, "shape"), f(self.id, "id")); - } - - std::string name() const { return "hip::hip_allocate_memory"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return s; - } - - argument compute(context& ctx, const shape&, const std::vector&) const - { - return get_preallocation(ctx, id); - } - - void finalize(context& ctx, const shape&, const std::vector&) const - { - argument a = allocate_gpu(s); - store_preallocated_param(ctx, id, a); - } -}; - -struct hip_copy_literal -{ - literal l; - std::string id{}; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.l, "literal"), f(self.id, "id")); - } - - std::string name() const { return "hip::hip_copy_literal"; } - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(0); - return l.get_shape(); - } - - argument compute(context& ctx, const shape&, const std::vector&) const - { - return get_preallocation(ctx, id); - } - - void finalize(context& ctx, const shape&, const std::vector&) const - { - argument a = to_gpu(l.get_argument()); - store_preallocated_param(ctx, id, a); - } - friend std::ostream& operator<<(std::ostream& os, const hip_copy_literal& x) - { - os << x.name() << "[id=" << x.id << "]"; - return os; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip_gemm.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip_gemm.hpp deleted file mode 100644 index 8c3d67bcd..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip_gemm.hpp +++ /dev/null @@ -1,146 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_HIP_GEMM_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_HIP_GEMM_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; -void blas_shape_hip(const shape& s); -shape transpose_batch_hip(const shape& s, unsigned trans_batch); - -template -struct hip_gemm -{ - Op op; - float alpha = 1; - float beta = 0; - unsigned trans_batch = 0; - int32_t solution_idx = 0; - - template - static auto reflect(Self& self, F f) - { - return pack_join(migraphx::reflect(self.op, f), - pack(f(self.alpha, "alpha"), - f(self.beta, "beta"), - f(self.trans_batch, "trans_batch"), - f(self.solution_idx, "solution_idx"))); - } - - std::string name() const - { - if(contains(op.name(), "quant_")) - { - return "gpu::hip_quant_gemm"; - } - return "gpu::hip_gemm"; - } - - shape compute_shape(const std::vector& inputs) const - { - std::vector in_shapes(inputs); - in_shapes.pop_back(); - in_shapes.pop_back(); - // When input shapes are A, B, C the GEMM equation is C  =  α AB+ β C where α, β are - // scalars - check_shapes{in_shapes, *this}.has(2, 3); - blas_shape_hip(inputs[0]); - blas_shape_hip(inputs[1]); - // if gemm and add are fused - if(in_shapes.size() > 2) - { - auto cmat_shape = in_shapes.back(); - check_shapes{{cmat_shape}, *this}.not_transposed().not_broadcasted(); - in_shapes.pop_back(); - blas_shape_hip(cmat_shape); - auto op_out_shape = op.compute_shape(in_shapes); - if(cmat_shape.lens() != op_out_shape.lens()) - { - MIGRAPHX_THROW(this->name() + " : dimension mismatch, operand C: {" + - to_string_range(cmat_shape.lens()) + - "}, cannot add to operand A * B: {" + - to_string_range(op_out_shape.lens()) + "}"); - } - if(cmat_shape.type() != op_out_shape.type()) - { - MIGRAPHX_THROW(this->name() + " : operand C type mismatch, operand C is of type: " + - to_string(cmat_shape.type()) + - ", it must be: " + to_string(op_out_shape.type())); - } - return transpose_batch_hip(op_out_shape, trans_batch); - } - - return transpose_batch_hip(op.compute_shape(in_shapes), trans_batch); - } - - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const - { - hip_gemm_compute(ctx, output_shape, args, alpha, beta, solution_idx); - return args.back(); - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } - - void finalize(context& ctx, const shape& output_shape, const std::vector& input_shapes) - { - if(solution_idx == 0) - solution_idx = hip_gemm_default_solution(ctx, output_shape, input_shapes); - if(enabled(MIGRAPHX_ENABLE_HIP_GEMM_TUNING{}) or ctx.get_exhaustive_tune_flag()) - { - solution_idx = - hip_gemm_finalize(ctx, output_shape, input_shapes, alpha, beta, solution_idx); - } - } - - value - compile(migraphx::context& ctx, const shape& output, const std::vector& input_shapes) - { - finalize(any_cast(ctx), output, input_shapes); - size_t ws = hip_gemm_workspace_size( - any_cast(ctx), output, input_shapes, alpha, beta, solution_idx); - return {{"workspace", ws}}; - } -}; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_RTGLIB_GPU_HIP_GEMM_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip_gemm_impl.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip_gemm_impl.hpp deleted file mode 100644 index f26d594d8..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hip_gemm_impl.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_HIP_GEMM_IMPL_HPP -#define MIGRAPHX_GUARD_RTGLIB_HIP_GEMM_IMPL_HPP - -#include -#include -#include - -// Set this environment variable to "true" to perform GEMM tuning even when the -// --exhaustive-tune option isn't set. Can be used to skip slow convolution tuning. -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_HIP_GEMM_TUNING); - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using milliseconds = std::chrono::duration; -using microseconds = std::chrono::duration; - -/** - * @brief Templated implementations of the compute() and finalize() methods of the Gemm operator. - * For each function there are overloads using either float or int32_t for the arguments - * alpha and beta. - * - * @param ctx . - * @param output_shape . - * @param args . - * @param alpha . - * @param beta . - */ -void hip_gemm_compute(context& ctx, - const shape& output_shape, - const std::vector& args, - float alpha, - float beta, - int32_t solution_idx); - -int32_t hip_gemm_finalize(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - float alpha, - float beta, - int32_t solution_idx); - -int32_t hip_gemm_default_solution(context& ctx, - const shape& output_shape, - const std::vector& input_shapes); - -size_t hip_gemm_workspace_size(context& ctx, - const shape& output_shape, - const std::vector& input_shapes, - float alpha, - float beta, - int32_t solution_idx); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hipblaslt.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hipblaslt.hpp deleted file mode 100644 index 49d41bf4d..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/hipblaslt.hpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_HIPBLASLT_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_HIPBLASLT_HPP -#include -#include -#include -#include -#if MIGRAPHX_USE_HIPBLASLT -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// TODO: Remove hipblas_status_to_string() function when hipblaslt -// provides an API for doing this in hipBLASLt. - -// Convert hipblas_status to string -inline const char* hipblas_status_to_string(hipblasStatus_t status) -{ - switch(status) - { - case HIPBLAS_STATUS_SUCCESS: return "HIPBLAS_STATUS_SUCCESS"; - case HIPBLAS_STATUS_NOT_INITIALIZED: return "HIPBLAS_STATUS_NOT_INITIALIZED"; - case HIPBLAS_STATUS_ALLOC_FAILED: return "HIPBLAS_STATUS_ALLOC_FAILED"; - case HIPBLAS_STATUS_INVALID_VALUE: return "HIPBLAS_STATUS_INVALID_VALUE"; - case HIPBLAS_STATUS_MAPPING_ERROR: return "HIPBLAS_STATUS_MAPPING_ERROR"; - case HIPBLAS_STATUS_EXECUTION_FAILED: return "HIPBLAS_STATUS_EXECUTION_FAILED"; - case HIPBLAS_STATUS_INTERNAL_ERROR: return "HIPBLAS_STATUS_INTERNAL_ERROR"; - case HIPBLAS_STATUS_NOT_SUPPORTED: return "HIPBLAS_STATUS_NOT_SUPPORTED"; - case HIPBLAS_STATUS_ARCH_MISMATCH: return "HIPBLAS_STATUS_ARCH_MISMATCH"; - case HIPBLAS_STATUS_HANDLE_IS_NULLPTR: return "HIPBLAS_STATUS_HANDLE_IS_NULLPTR"; - case HIPBLAS_STATUS_INVALID_ENUM: return "HIPBLAS_STATUS_INVALID_ENUM"; - case HIPBLAS_STATUS_UNKNOWN: return "HIPBLAS_STATUS_UNKNOWN"; - } - return ""; -} - -template -inline auto hipblaslt_invoke(F f, Ts... xs) -{ - // Call the function `f` with `xs...` and capture the status - auto status = f(xs...); - - if(status != HIPBLAS_STATUS_SUCCESS) - { - std::string error_message = - "hipBLAS error: '" + std::string(hipblas_status_to_string(status)) + "'(" + - std::to_string(status) + ") at " + __FILE__ + ":" + std::to_string(__LINE__); - MIGRAPHX_THROW(EXIT_FAILURE, error_message); - } - return status; -} - -// Invoke a hipBLASLt call. If used to validate a call, set fatal_error = false to prevent -// throwing an exception on failure. -template -auto hipblaslt_invoke(F f, Pack p, Ts... xs, bool fatal_error = true) -{ - return p([=](auto... ws) { - auto status = f(ws..., xs...); - if(status != HIPBLAS_STATUS_SUCCESS) - { - if(fatal_error) - { - MIGRAPHX_THROW("hipblaslt_invoke: hipBlasLt call failed with status " + - std::to_string(status)); - } - } - return status; - }); -} - -using hipblaslt_handle_ptr = MIGRAPHX_MANAGE_PTR(hipblasLtHandle_t, hipblasLtDestroy); -using hipblaslt_preference_ptr = MIGRAPHX_MANAGE_PTR(hipblasLtMatmulPreference_t, - hipblasLtMatmulPreferenceDestroy); - -hipblaslt_handle_ptr create_hipblaslt_handle_ptr(); -hipblaslt_preference_ptr create_hipblaslt_preference_ptr(); -bool hipblaslt_supported(); -const size_t hipblaslt_workspace_size = 2 * 128 * 1024 * 1024; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_USE_HIPBLASLT -#endif // MIGRAPHX_GUARD_MIGRAPHLIB_HIPBLASLT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/kernel.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/kernel.hpp deleted file mode 100644 index 63accdea4..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/kernel.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_KERNEL_HPP -#define MIGRAPHX_GUARD_RTGLIB_KERNEL_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct kernel_impl; - -struct MIGRAPHX_GPU_EXPORT kernel -{ - kernel() = default; - kernel(const char* image, const std::string& name); - template - kernel(const std::vector& image, const std::string& name) - : kernel(reinterpret_cast(image.data()), name) - { - } - - void launch(hipStream_t stream, - std::size_t global, - std::size_t local, - const std::vector& args, - hipEvent_t start = nullptr, - hipEvent_t stop = nullptr) const; - - void launch(hipStream_t stream, - std::size_t global, - std::size_t local, - std::vector args, - hipEvent_t start = nullptr, - hipEvent_t stop = nullptr) const; - - template - auto launch(hipStream_t stream, std::size_t global, std::size_t local, Ts... zs) const - { - return [=](auto&&... xs) { - launch(stream, global, local, std::vector{xs...}, zs...); - }; - } - - private: - std::shared_ptr impl; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/logsoftmax.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/logsoftmax.hpp deleted file mode 100644 index 5ea23ee27..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/logsoftmax.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_LOGSOFTMAX_HPP -#define MIGRAPHX_GUARD_RTGLIB_LOGSOFTMAX_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct hip_logsoftmax -{ - op::logsoftmax op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::logsoftmax"; } - shape compute_shape(const std::vector& inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/loop.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/loop.hpp deleted file mode 100644 index 792c84b74..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/loop.hpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_LOOP_HPP -#define MIGRAPHX_GUARD_RTGLIB_LOOP_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_loop -{ - op::loop op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::loop"; } - shape compute_shape(std::vector inputs, std::vector mods) const; - argument - compute(context& ctx, - const shape& output_shape, - const std::vector& args, - const std::vector& mods, - const std::function( - module_ref&, const std::unordered_map&)>& run) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/lowering.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/lowering.hpp deleted file mode 100644 index 6f4a3ca3e..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/lowering.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MIOPEN_LOWERING_HPP -#define MIGRAPHX_GUARD_RTGLIB_MIOPEN_LOWERING_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -namespace gpu { - -/** - * Compiler pass that makes GPU-specific instruction changes. - * * Copies to and from the device if `offload_copy` is true. - * * Maps instructions to their GPU-specific counterparts. - * * Inserts `allocate` instructions before GPU operators. - */ -struct MIGRAPHX_GPU_EXPORT lowering -{ - context* ctx; - bool offload_copy; - std::string name() const { return "gpu::lowering"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/lrn.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/lrn.hpp deleted file mode 100644 index 8ccda7bba..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/lrn.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_LRN_HPP -#define MIGRAPHX_GUARD_RTGLIB_LRN_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; -#if MIGRAPHX_USE_MIOPEN -struct miopen_lrn -{ - op::lrn op; - shared ldesc; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::lrn"; } - shape compute_shape(const std::vector& inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - void finalize(context&, const shape&, const std::vector&); - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/miopen.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/miopen.hpp deleted file mode 100644 index 87a561ad6..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/miopen.hpp +++ /dev/null @@ -1,343 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_MIOPEN_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_MIOPEN_HPP - -#include -#include -#include -#if MIGRAPHX_USE_MIOPEN -#include -#include -#include -#include - -#include - -#ifdef MIGRAPHX_HAS_FIND_MODE_API -extern "C" miopenStatus_t -miopenHiddenSetConvolutionFindMode(miopenConvolutionDescriptor_t convDesc, // NOLINT - int findMode); // NOLINT -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -using miopen_handle = MIGRAPHX_MANAGE_PTR(miopenHandle_t, miopenDestroy); -using tensor_descriptor = MIGRAPHX_MANAGE_PTR(miopenTensorDescriptor_t, - miopenDestroyTensorDescriptor); -using convolution_descriptor = MIGRAPHX_MANAGE_PTR(miopenConvolutionDescriptor_t, - miopenDestroyConvolutionDescriptor); -using pooling_descriptor = MIGRAPHX_MANAGE_PTR(miopenPoolingDescriptor_t, - miopenDestroyPoolingDescriptor); -using activation_descriptor = MIGRAPHX_MANAGE_PTR(miopenActivationDescriptor_t, - miopenDestroyActivationDescriptor); -using fusion_plan_descriptor = MIGRAPHX_MANAGE_PTR(miopenFusionPlanDescriptor_t, - miopenDestroyFusionPlan); -using fused_operator_args = MIGRAPHX_MANAGE_PTR(miopenOperatorArgs_t, miopenDestroyOperatorArgs); - -using lrn_descriptor = MIGRAPHX_MANAGE_PTR(miopenLRNDescriptor_t, miopenDestroyLRNDescriptor); - -template -Result make_obj(F f, Ts... xs) -{ - typename Result::pointer x = nullptr; - auto status = f(&x, xs...); - Result r{x}; - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MAKE_OBJ: MIOpen call failed"); - return r; -} - -#ifdef MIGRAPHX_HAS_FIND_2_API -using miopen_find_options = MIGRAPHX_MANAGE_PTR(miopenFindOptions_t, miopenDestroyFindOptions); -using miopen_problem = MIGRAPHX_MANAGE_PTR(miopenProblem_t, miopenDestroyProblem); -using miopen_solution = MIGRAPHX_MANAGE_PTR(miopenSolution_t, miopenDestroySolution); - -inline miopen_solution find_solution(miopenHandle_t handle, - size_t num_inputs, - const miopenTensorArgument_t* tensor_args, - void* workspace, - size_t workspace_size, - miopenProblem_t problem, - bool tune = false) -{ - miopenSolution_t solution; - size_t found = 0; - miopen_find_options fo = make_obj(&miopenCreateFindOptions); - if(tune) - { - miopenSetFindOptionTuning(fo.get(), 1); - } -#ifdef MIGRAPHX_PREALLOCATE_MIOPEN_BUFFERS - for(auto i : range(num_inputs)) - { - auto status = miopenSetFindOptionPreallocatedTensor( - fo.get(), tensor_args[i].id, tensor_args[i].buffer); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen: failed to preallocate tensors for the find process"); - } - auto status = miopenSetFindOptionPreallocatedWorkspace(fo.get(), workspace, workspace_size); - if(status != miopenStatusSuccess) - MIGRAPHX_THROW("MIOpen: failed to preallocate workspace for the find process"); -#else - miopenStatus_t status; - (void)(num_inputs); - (void)(tensor_args); - (void)(workspace_size); - (void)(workspace); -#endif - status = miopenFindSolutions(handle, problem, fo.get(), &solution, &found, 1); - auto result = miopen_solution{solution}; - if(status != miopenStatusSuccess or found == 0) - MIGRAPHX_THROW("MIOpen: miopenFindSolutions failed"); - return result; -} - -inline void set_tensor_descriptor(miopenTensorArgumentId_t name, - tensor_descriptor& desc, - miopen_problem& problem_ptr) -{ - auto status = miopenSetProblemTensorDescriptor(problem_ptr.get(), name, desc.get()); - if(status != miopenStatusSuccess) - { - MIGRAPHX_THROW("setting problem tensor description failed"); - } -} -#endif - -inline tensor_descriptor make_tensor(const migraphx::shape& os) -{ - auto s = os.normalize_standard(); - auto t = make_obj(&miopenCreateTensorDescriptor); - // Convert to ints - std::vector lens(s.lens().begin(), s.lens().end()); - std::vector strides(s.strides().begin(), s.strides().end()); - miopenDataType_t d; - if(s.type() == shape::float_type) - d = miopenFloat; - else if(s.type() == shape::half_type) - d = miopenHalf; - else if(s.type() == shape::int32_type) - d = miopenInt32; - else if(s.type() == shape::int8_type) - d = miopenInt8; - else if(s.type() == shape::bf16_type) - d = miopenBFloat16; - else - MIGRAPHX_THROW("MAKE_TENSOR: unsupported type"); - miopenSetTensorDescriptor(t.get(), d, s.lens().size(), lens.data(), strides.data()); - - return t; -} - -template -inline convolution_descriptor make_conv(const T& op) -{ - auto c = make_obj(&miopenCreateConvolutionDescriptor); - miopenConvolutionMode_t c_mode = miopenConvolution; - if(op.group > 1) - c_mode = miopenGroupConv; - - int kdims = op.kdims(); - std::vector padding(std::max(2, kdims), 0); - std::vector stride(std::max(2, kdims), 1); - std::vector dilation(std::max(2, kdims), 1); - - std::copy_backward(op.padding.begin(), op.padding.begin() + kdims, padding.end()); - std::copy_backward(op.stride.begin(), op.stride.end(), stride.end()); - std::copy_backward(op.dilation.begin(), op.dilation.end(), dilation.end()); - - miopenInitConvolutionNdDescriptor( - c.get(), padding.size(), padding.data(), stride.data(), dilation.data(), c_mode); - if(op.group > 1) - miopenSetConvolutionGroupCount(c.get(), op.group); -#ifdef MIGRAPHX_HAS_FIND_MODE_API - miopenHiddenSetConvolutionFindMode(c.get(), 1); // Normal mode -#endif - return c; -} - -template -inline convolution_descriptor make_convolution_backwards(const T& op) -{ - auto c = make_obj(&miopenCreateConvolutionDescriptor); - miopenConvolutionMode_t c_mode = miopenTranspose; - int kdims = op.kdims(); - std::vector padding(std::max(2, kdims), 0); - std::vector stride(std::max(2, kdims), 1); - std::vector dilation(std::max(2, kdims), 1); - - std::copy_backward(op.padding.begin(), op.padding.end(), padding.end()); - std::copy_backward(op.stride.begin(), op.stride.end(), stride.end()); - std::copy_backward(op.dilation.begin(), op.dilation.end(), dilation.end()); - - miopenInitConvolutionNdDescriptor( - c.get(), padding.size(), padding.data(), stride.data(), dilation.data(), c_mode); - if(op.group > 1) - miopenSetConvolutionGroupCount(c.get(), op.group); - return c; -} - -inline pooling_descriptor make_pooling(const migraphx::op::pooling& op) -{ - miopenPoolingMode_t mode; - if(op.mode == op::pooling_mode::max) - mode = miopenPoolingMax; - else if(op.mode == op::pooling_mode::average) - mode = miopenPoolingAverage; - else - { - std::stringstream ss("Unknown mode for pooling: "); - ss << op.mode; - MIGRAPHX_THROW(ss.str()); - } - if(not std::all_of( - op.dilations.cbegin(), op.dilations.cend(), [](std::size_t d) { return d == 1; })) - { - MIGRAPHX_THROW("Unsupported dilations for pooling: [" + to_string_range(op.dilations) + - "]"); - } - auto p = make_obj(&miopenCreatePoolingDescriptor); - - int kdims = op.kdims(); - std::vector padding(std::max(2, kdims), 0); - std::vector stride(std::max(2, kdims), 1); - std::vector lengths(std::max(2, kdims), 1); - - std::copy_backward(op.padding.begin(), op.padding.begin() + kdims, padding.end()); - std::copy_backward(op.stride.begin(), op.stride.end(), stride.end()); - std::copy_backward(op.lengths.begin(), op.lengths.end(), lengths.end()); - - miopenSetNdPoolingDescriptor( - p.get(), mode, padding.size(), lengths.data(), padding.data(), stride.data()); - return p; -} - -inline lrn_descriptor make_lrn(const migraphx::op::lrn& op) -{ - auto ldesc = make_obj(&miopenCreateLRNDescriptor); - miopenSetLRNDescriptor(ldesc.get(), miopenLRNCrossChannel, op.size, op.alpha, op.beta, op.bias); - return ldesc; -} - -inline activation_descriptor make_relu() -{ - auto ad = make_obj(&miopenCreateActivationDescriptor); - miopenSetActivationDescriptor(ad.get(), miopenActivationRELU, 0, 0, 0); - return ad; -} - -inline activation_descriptor make_sigmoid() -{ - auto ad = make_obj(&miopenCreateActivationDescriptor); - miopenSetActivationDescriptor(ad.get(), miopenActivationLOGISTIC, 0, 0, 0); - return ad; -} - -inline activation_descriptor make_tanh() -{ - auto ad = make_obj(&miopenCreateActivationDescriptor); - // onnx operator does not apply additional scaling for tanh - // defaults for alpha and beta are therefore set to 1 - miopenSetActivationDescriptor(ad.get(), miopenActivationTANH, 1, 1, 0); - return ad; -} - -inline activation_descriptor make_abs() -{ - auto ad = make_obj(&miopenCreateActivationDescriptor); - miopenSetActivationDescriptor(ad.get(), miopenActivationABS, 0, 0, 0); - return ad; -} - -inline activation_descriptor make_leaky_relu(double alpha) -{ - auto ad = make_obj(&miopenCreateActivationDescriptor); - miopenSetActivationDescriptor(ad.get(), miopenActivationLEAKYRELU, alpha, 0, 0); - return ad; -} - -inline activation_descriptor make_elu(double alpha) -{ - auto ad = make_obj(&miopenCreateActivationDescriptor); - miopenSetActivationDescriptor(ad.get(), miopenActivationELU, alpha, 0, 0); - return ad; -} - -inline fusion_plan_descriptor make_fusion_plan(const shape& input) -{ - auto t = make_tensor(input); - return make_obj(&miopenCreateFusionPlan, miopenVerticalFusion, t.get()); -} - -// Temporary hack to workaround memory problems in miopen -inline fusion_plan_descriptor make_fusion_plan(const tensor_descriptor& input) -{ - return make_obj( - &miopenCreateFusionPlan, miopenVerticalFusion, input.get()); -} - -inline fused_operator_args make_fused_args() -{ - return make_obj(&miopenCreateOperatorArgs); -} - -template -auto reflect(miopenActivationDescriptor_t ad, F f) -{ - assert(ad != nullptr); - miopenActivationMode_t mode = miopenActivationPASTHRU; - double alpha = 0.0; - double beta = 0.0; - double gamma = 0.0; - miopenGetActivationDescriptor(ad, &mode, &alpha, &beta, &gamma); - return pack(f(std::move(mode), "mode"), // NOLINT - f(std::move(alpha), "alpha"), // NOLINT - f(std::move(beta), "beta"), // NOLINT - f(std::move(gamma), "gamma")); // NOLINT -} - -template -auto reflect(miopenLRNDescriptor_t lrnd, F f) -{ - assert(lrnd != nullptr); - miopenLRNMode_t mode = miopenLRNWithinChannel; - unsigned int n = 0; - double alpha = 0.0; - double beta = 0.0; - double k = 0.0; - miopenGetLRNDescriptor(lrnd, &mode, &n, &alpha, &beta, &k); - return pack(f(std::move(mode), "mode"), // NOLINT - f(std::move(n), "n"), // NOLINT - f(std::move(alpha), "alpha"), // NOLINT - f(std::move(beta), "beta"), // NOLINT - f(std::move(k), "k")); // NOLINT -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/mlir.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/mlir.hpp deleted file mode 100644 index d1f19c1e8..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/mlir.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_MLIR_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_MLIR_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct module; -namespace gpu { - -MIGRAPHX_GPU_EXPORT std::string dump_mlir(module m); -MIGRAPHX_GPU_EXPORT std::string dump_mlir(module m, const std::vector& inputs); -MIGRAPHX_GPU_EXPORT void -dump_mlir_to_file(module m, const std::vector& inputs, const fs::path& location); - -MIGRAPHX_GPU_EXPORT bool -is_module_fusible(const module& m, const context& migraphx_ctx, const value& solution); - -struct MIGRAPHX_GPU_EXPORT mlir_code_object -{ - code_object_op cop; - std::vector prefill_indices = {}; - std::vector prefill_values = {}; -}; - -MIGRAPHX_GPU_EXPORT bool is_reduce(const instruction& ins); - -MIGRAPHX_GPU_EXPORT mlir_code_object compile_mlir(const context& migraphx_ctx, - module m, - const std::vector& in_shapes, - const value& solution); - -MIGRAPHX_GPU_EXPORT instruction_ref insert_mlir(module& m, - instruction_ref ins, - code_object_op co, - const std::vector& inputs); - -MIGRAPHX_GPU_EXPORT tuning_config get_tuning_config_mlir(const context& migraphx_ctx, - module m, - const std::vector& inputs, - bool exhaustive); - -MIGRAPHX_GPU_EXPORT void -dump_mlir_to_mxr(module m, const std::vector& inputs, const fs::path& location); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/multinomial.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/multinomial.hpp deleted file mode 100644 index c44d48082..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/multinomial.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MULTINOMIAL_HPP -#define MIGRAPHX_GUARD_RTGLIB_MULTINOMIAL_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_multinomial -{ - op::multinomial op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::multinomial"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/name.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/name.hpp deleted file mode 100644 index 390d7ea0b..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/name.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_OP_NAME_HPP -#define MIGRAPHX_GUARD_RTGLIB_OP_NAME_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -template -struct oper -{ - // function to extract the name part of an operator. For example, we have - // a operation "sin", then the get_type_name() will return - // "migraphx::version_1::gpu::hip_sin", this functin will return the name - // "gpu::sin" as the operator name - std::string name() const - { - const std::string& name = get_type_name(); - // search the namespace gpu (::gpu::) - auto pos_ns = name.find("::gpu::"); - if(pos_ns != std::string::npos) - { - auto pos_name = name.find("hip_", pos_ns + std::string("::gpu::").length()); - if(pos_name != std::string::npos) - { - return std::string("gpu::") + name.substr(pos_name + 4); - } - else - { - return name.substr(pos_ns + 2); - } - } - return "unknown_operator_name"; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/nonzero.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/nonzero.hpp deleted file mode 100644 index cfc7e78db..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/nonzero.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_NONZERO_HPP -#define MIGRAPHX_GUARD_RTGLIB_NONZERO_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_nonzero -{ - op::nonzero op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::nonzero"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/oper.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/oper.hpp deleted file mode 100644 index 13ac11a3d..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/oper.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_UNARY_HPP -#define MIGRAPHX_GUARD_RTGLIB_UNARY_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -template -struct device_base : oper -{ - template - static auto reflect(Self&, F) - { - return pack(); - } - - std::vector reduce_shapes; - - void finalize(context&, const shape&, const std::vector& inputs) - { - reduce_shapes = reduce_dims(inputs); - } - - argument get_arg(const std::vector& args, std::size_t i) const - { - if(reduce_shapes.empty()) - return args[i]; - return args.at(i).reshape(reduce_shapes.at(i)); - } - - shape compute_shape(const std::vector& inputs) const - { - check_shapes{inputs, *this}.has(N + 1); - auto s0 = inputs.at(0); - if(std::all_of(inputs.begin(), inputs.end() - 1, [&](auto s) { return s == s0; }) and - s0.packed()) - return s0; - else - return {s0.type(), s0.lens()}; - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -template -struct unary_device : device_base -{ - argument compute(context& ctx, const shape&, const std::vector& args) const - { - F(ctx.get_stream().get(), this->get_arg(args, 1), this->get_arg(args, 0)); - return args[1]; - } -}; - -template -struct binary_device : device_base -{ - argument compute(context& ctx, const shape&, const std::vector& args) const - { - F(ctx.get_stream().get(), - this->get_arg(args, 2), - this->get_arg(args, 0), - this->get_arg(args, 1)); - return args[2]; - } -}; - -template -struct ternary_device : device_base -{ - argument compute(context& ctx, const shape&, const std::vector& args) const - { - F(ctx.get_stream().get(), - this->get_arg(args, 3), - this->get_arg(args, 0), - this->get_arg(args, 1), - this->get_arg(args, 2)); - return args[3]; - } -}; - -template -struct quaternary_device : device_base -{ - argument compute(context& ctx, const shape&, const std::vector& args) const - { - F(ctx.get_stream().get(), - this->get_arg(args, 4), - this->get_arg(args, 0), - this->get_arg(args, 1), - this->get_arg(args, 2), - this->get_arg(args, 3)); - return args[4]; - } -}; - -template -struct quinary_device : device_base -{ - argument compute(context& ctx, const shape&, const std::vector& args) const - { - F(ctx.get_stream().get(), - this->get_arg(args, 5), - this->get_arg(args, 0), - this->get_arg(args, 1), - this->get_arg(args, 2), - this->get_arg(args, 3), - this->get_arg(args, 4)); - return args[5]; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/pack_args.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/pack_args.hpp deleted file mode 100644 index 1896a3008..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/pack_args.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_PACK_ARGS_HPP -#define MIGRAPHX_GUARD_RTGLIB_PACK_ARGS_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct kernel_argument -{ - template , - MIGRAPHX_REQUIRES(not std::is_base_of{})> - kernel_argument(T&& x) : size(sizeof(U)), align(alignof(U)), data(&x) // NOLINT - { - } - std::size_t size; - std::size_t align; - void* data; -}; - -MIGRAPHX_GPU_EXPORT std::vector pack_args(const std::vector& args); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/perfdb.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/perfdb.hpp deleted file mode 100644 index 21aed313c..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/perfdb.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_PERFDB_HPP -#define MIGRAPHX_GUARD_GPU_PERFDB_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct problem_params -{ - operation op; - std::vector inputs; - shape output; -}; - -std::string get_mlir_perf_for_conv(const problem_params& pp, bool xdlops); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_PERFDB_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/pooling.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/pooling.hpp deleted file mode 100644 index 7f6722b11..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/pooling.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_POOLING_HPP -#define MIGRAPHX_GUARD_RTGLIB_POOLING_HPP - -#include -#include -#include -#include -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; -#if MIGRAPHX_USE_MIOPEN -struct miopen_pooling -{ - op::pooling op; - shared pd; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::pooling"; } - shape compute_shape(const std::vector& inputs) const; - void finalize(context&, const shape&, const std::vector&); - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; -#endif - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prefix_scan_sum.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prefix_scan_sum.hpp deleted file mode 100644 index cca8efd60..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prefix_scan_sum.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_PREFIX_SCAN_SUM_HPP -#define MIGRAPHX_GUARD_GPU_PREFIX_SCAN_SUM_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_prefix_scan_sum : oper -{ - op::prefix_scan_sum op; - - template - static auto reflect(Self& self, T f) - { - return migraphx::reflect(self.op, f); - } - - shape compute_shape(const std::vector& inputs) const - { - std::vector in_shapes{inputs}; - in_shapes.pop_back(); - check_shapes{in_shapes, *this}.standard(); - return op.normalize_compute_shape(in_shapes); - } - - argument compute(context& ctx, const shape&, const std::vector& args) const - { - device::prefix_scan_sum( - ctx.get_stream().get(), args[1], args[0], op.axis, op.exclusive, op.reverse); - return args[1]; - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_PREFIX_SCAN_SUM_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prefuse_ops.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prefuse_ops.hpp deleted file mode 100644 index bed640520..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prefuse_ops.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_PREFUSE_OPS_HPP -#define MIGRAPHX_GUARD_GPU_PREFUSE_OPS_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module_pass_manager; - -namespace gpu { - -struct MIGRAPHX_GPU_EXPORT prefuse_ops -{ - bool enable_attention = false; - std::string name() const { return "gpu::prefuse_ops"; } - void apply(module_pass_manager& mpm) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif // MIGRAPHX_GUARD_GPU_PREFUSE_OPS_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prepare_reduce.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prepare_reduce.hpp deleted file mode 100644 index 3c6bfdd42..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/prepare_reduce.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_GPU_PREPARE_REDUCE_HPP -#define MIGRAPHX_GUARD_GPU_PREPARE_REDUCE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; - -namespace gpu { - -struct prepare_reduce -{ - std::string name() const { return "gpu::prepare_reduce"; } - void apply(module& m) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_PREPARE_REDUCE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/problem_cache.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/problem_cache.hpp deleted file mode 100644 index d70e0687b..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/problem_cache.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_GPU_PROBLEM_CACHE_HPP -#define MIGRAPHX_GUARD_GPU_PROBLEM_CACHE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -struct MIGRAPHX_GPU_EXPORT problem_cache -{ - bool has(const std::string& name, const value& problem) const; - void insert(const std::string& name, const value& problem, const value& solution); - void mark(const std::string& name, const value& problem); - optional get(const std::string& name, const value& problem) const; - void load(); - void save() const; - std::unordered_map cache; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_PROBLEM_CACHE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/reduce_op.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/reduce_op.hpp deleted file mode 100644 index 10f3dcf84..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/reduce_op.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REDUCE_OP_HPP -#define MIGRAPHX_GUARD_RTGLIB_REDUCE_OP_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -template -struct reduce_op : oper -{ - Op op; - - template - static auto reflect(Self& self, T f) - { - return migraphx::reflect(self.op, f); - } - - shape compute_shape(const std::vector& inputs) const - { - std::vector in_shapes{inputs}; - in_shapes.pop_back(); - check_shapes{in_shapes, *this}.standard(); - return op.normalize_compute_shape(in_shapes); - } - - argument compute(context& ctx, const shape&, const std::vector& args) const - { - F(ctx.get_stream().get(), args[1], args[0]); - return args[1]; - } - - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } - - reduce_op() {} - reduce_op(const Op& op_ref) : op(op_ref) {} -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/reverse.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/reverse.hpp deleted file mode 100644 index 8ef825235..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/reverse.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_REVERSE_HPP -#define MIGRAPHX_GUARD_RTGLIB_REVERSE_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_reverse -{ - op::reverse op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::reverse"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/rnn_variable_seq_lens.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/rnn_variable_seq_lens.hpp deleted file mode 100644 index 7d811192d..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/rnn_variable_seq_lens.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_RNN_VARIABLE_SEQ_LENS_HPP -#define MIGRAPHX_GUARD_RTGLIB_RNN_VARIABLE_SEQ_LENS_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct hip_rnn_var_sl_shift_sequence -{ - op::rnn_var_sl_shift_sequence op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::rnn_var_sl_shift_sequence"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -struct hip_rnn_var_sl_shift_output -{ - op::rnn_var_sl_shift_output op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::rnn_var_sl_shift_output"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -struct hip_rnn_var_sl_last_output -{ - op::rnn_var_sl_last_output op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::" + op.name(); } - shape compute_shape(std::vector inputs) const; - argument compute(context& ctx, const shape&, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/rocblas.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/rocblas.hpp deleted file mode 100644 index d23c40f9d..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/rocblas.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_ROCBLAS_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_ROCBLAS_HPP -#include -#include -#if MIGRAPHX_USE_ROCBLAS -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_ROCBLAS - -using rocblas_handle_ptr = MIGRAPHX_MANAGE_PTR(rocblas_handle, rocblas_destroy_handle); - -rocblas_handle_ptr create_rocblas_handle_ptr(); -rocblas_handle_ptr create_rocblas_handle_ptr(hipStream_t s); -#endif -struct context; - -MIGRAPHX_GPU_EXPORT bool get_compute_fp32_flag(); - -MIGRAPHX_GPU_EXPORT bool rocblas_fp8_available(); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/schedule_model.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/schedule_model.hpp deleted file mode 100644 index d9c692cb7..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/schedule_model.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_SCHEDULE_MODEL_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_SCHEDULE_MODEL_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct module; -struct operation; - -namespace gpu { - -struct schedule_model -{ - std::size_t streams = 0; - std::size_t concurrency() const; - void sched(module& m, instruction_ref ins, std::size_t n) const; - void wait(module& m, instruction_ref ins, std::size_t wait_id) const; - void record(module& m, instruction_ref ins, std::size_t wait_id) const; - std::size_t weight(const operation& op) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/sync_device.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/sync_device.hpp deleted file mode 100644 index 331152cbf..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/sync_device.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_GPU_SYNC_DEVICE_HPP -#define MIGRAPHX_GUARD_RTGLIB_GPU_SYNC_DEVICE_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct module; - -namespace gpu { - -struct sync_device -{ - std::string name() const { return "sync_device"; } - void apply(module& m) const; -}; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/target.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/target.hpp deleted file mode 100644 index 407c44fec..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/target.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_MIOPEN_TARGET_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_MIOPEN_TARGET_HPP - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct MIGRAPHX_GPU_EXPORT target -{ - std::string name() const; - std::vector get_passes(migraphx::context& gctx, const compile_options& options) const; - migraphx::context get_context() const; - argument copy_to(const argument& arg) const; - argument copy_from(const argument& arg) const; - argument allocate(const shape& s) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/time_op.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/time_op.hpp deleted file mode 100644 index 2c5893eed..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/time_op.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_DRIVER_PERF_HPP -#define MIGRAPHX_GUARD_GPU_DRIVER_PERF_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_GPU_EXPORT double -time_op(const context& ictx, operation op, const std::vector& inputs, int n = 100); - -MIGRAPHX_GPU_EXPORT double time_program(const context& ictx, program p, int n = 100); - -/* benchmark gpu::code_object with expected input shapes over n iterations */ -MIGRAPHX_GPU_EXPORT double time_op(const context& ictx, operation op, int n = 100); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_DRIVER_PERF_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/topk.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/topk.hpp deleted file mode 100644 index f1df9d469..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/topk.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_TOPK_HPP -#define MIGRAPHX_GUARD_RTGLIB_TOPK_HPP - -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct context; - -struct hip_topk -{ - op::topk op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "gpu::topk"; } - shape compute_shape(std::vector inputs) const; - argument - compute(context& ctx, const shape& output_shape, const std::vector& args) const; - std::ptrdiff_t output_alias(const std::vector& shapes) const - { - return shapes.size() - 1; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/tuning_config.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/tuning_config.hpp deleted file mode 100644 index 23538c0d3..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/tuning_config.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_GPU_TUNING_CONFIG_HPP -#define MIGRAPHX_GUARD_GPU_TUNING_CONFIG_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct tuning_config -{ - value problem; - std::vector solutions; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif // MIGRAPHX_GUARD_GPU_TUNING_CONFIG_HPP diff --git a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/write_literals.hpp b/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/write_literals.hpp deleted file mode 100644 index 85a2ce3a8..000000000 --- a/docker/rocm/migraphx/targets/gpu/include/migraphx/gpu/write_literals.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_MIOPEN_WRITE_LITERALS_HPP -#define MIGRAPHX_GUARD_RTGLIB_MIOPEN_WRITE_LITERALS_HPP - -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct module; - -namespace gpu { - -struct MIGRAPHX_GPU_EXPORT write_literals -{ - context* ctx = nullptr; - std::string name() const { return "gpu::write_literals"; } - - void apply(module& m) const; -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/jit/ck_gemm.cpp b/docker/rocm/migraphx/targets/gpu/jit/ck_gemm.cpp deleted file mode 100644 index 392eaa0c6..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/ck_gemm.cpp +++ /dev/null @@ -1,235 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const ck_gemm_kernel = R"__migraphx__( -#include -#include -#include -#include -#include <${include}> - -namespace migraphx { - -${preamble} - -extern "C" { - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - ck_gemm<${solution}, ${blocks_per_batch}>(xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct ck_gemm_compiler : compiler -{ - std::vector names() const { return {"ck_gemm", "gpu::ck_gemm"}; } - - ck::host::device_gemm_multiple_d::Problem create_problem(const std::vector& inputs, - const value& v) const - { - const auto& a_shape = inputs[0]; - const auto& b_shape = inputs[1]; - const auto& c_shape = inputs.back(); - - // cppcheck-suppress unreadVariable - auto rank = a_shape.ndim(); - auto batch_count = get_batch_count(c_shape); - auto m = c_shape.lens()[rank - 2]; - m = can_fold_batch(inputs) ? m * batch_count : m; - auto n = c_shape.lens().back(); - auto k = a_shape.lens().back(); - - const bool trans_a = transposed_matrix(a_shape); - const bool trans_b = transposed_matrix(b_shape); - const bool trans_e = transposed_matrix(c_shape); - const auto a_type = get_type(a_shape); - const auto b_type = get_type(b_shape); - const auto e_type = get_type(c_shape); - std::vector ds_layout; - std::transform(inputs.begin() + 2, - inputs.end() - 1, - std::back_inserter(ds_layout), - [](const auto& i) { return transposed_matrix(i); }); - std::vector ds_type; - std::transform(inputs.begin() + 2, - inputs.end() - 1, - std::back_inserter(ds_type), - [](const auto& i) { return get_type(i); }); - - std::string ck_passthrough = "ck_passthrough"; - std::string cde_op = ck_passthrough; - assert(inputs.size() < 4 or v.contains("post")); - if(v.contains("post")) - { - cde_op = v.at("post").to(); - } - - return ck::host::device_gemm_multiple_d::Problem{m, - n, - k, - trans_a, - trans_b, - trans_e, - ds_layout, - a_type, - b_type, - e_type, - ds_type, - ck_passthrough, - ck_passthrough, - cde_op}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - const auto& c_shape = inputs.back(); - auto tuning_value = v.get("tuning_value", 34); - auto batch_count = get_batch_count(c_shape); - auto problem = create_problem(inputs, v); - - const auto include_header = problem.GetIncludeHeader(); - const auto solutions = problem.GetSolutions(ctx.get_current_device().get_gfx_name()); - const auto& solution = solutions.at(tuning_value); - const auto template_str = solution.template_str; - const auto blocks_per_batch = solution.grid_size; - const auto block_size = solution.block_size; - - hip_compile_options options; - options.additional_src_files = ck_headers(); - auto grid_size = can_fold_batch(inputs) ? blocks_per_batch : batch_count * blocks_per_batch; - options.set_launch_params(v, grid_size * block_size, block_size); - options.inputs = inputs; - options.output = c_shape; - options.kernel_name = v.get("kernel", "ck_gemm_kernel"); - options.virtual_inputs = inputs; - if(can_fold_batch(inputs)) - { - auto vinputs = inputs; - fold_batch_dims(vinputs[0]); - remove_batch_dims(vinputs[1]); - std::for_each(vinputs.begin() + 2, vinputs.end(), fold_batch_dims); - options.virtual_inputs = vinputs; - } - - if(v.get("check", false) or enabled(MIGRAPHX_CK_DEBUG{})) - options.emplace_param("-DMIGRAPHX_CK_CHECK=1"); - - auto src = interpolate_string(ck_gemm_kernel, - {{"solution", template_str}, - {"include", include_header}, - {"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"blocks_per_batch", to_string(blocks_per_batch)}, - {"preamble", v.get("preamble", std::string{})}, - {"kernel", options.kernel_name}}); - - return compile_hip_code_object(ctx, src, options); - } - - value create_settings(instruction_ref ins, const operation& op) const - { - auto v = op.to_value(); - v["kernel"] = "ck_gemm_kernel"; - if(not ins->module_inputs().empty()) - { - auto* pm = ins->module_inputs().front(); - v["preamble"] = generate_pointwise(*pm, "post_ck_gemm_function") + - "\nMIGRAPHX_LIFT_CLASS(post_ck_gemm, post_ck_gemm_function);"; - v["post"] = "ck_function_adaptor"; - v["kernel"] = to_c_id("ck_gemm_" + generate_name_from_ops(*pm) + "_kernel"); - } - return v; - } - - compiler_replace - compile(context& ctx, instruction_ref ins, const operation& op, const value& solution) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = create_settings(ins, op); - if(not solution.is_null()) - v["tuning_value"] = solution; - return {compile_op(ctx, shapes, v), - [=](module& m, instruction_ref ins2, const operation& code_object) { - if(enabled(MIGRAPHX_LOG_CK_GEMM{})) - { - std::vector gemm_shapes{ - shapes[0], shapes[1], shapes.back().with_type(shapes[0].type())}; - std::cout << "gpu::ck_gemm: " << to_json_string(to_value(gemm_shapes)) - << std::endl; - } - m.replace_instruction(ins2, code_object, ins2->inputs()); - }}; - } - - optional - get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool exhaustive) const - { - if(not exhaustive and not enabled(MIGRAPHX_TUNE_CK{})) - return nullopt; - tuning_config tc; - auto shapes = to_shapes(ins->inputs()); - auto problem = create_problem(shapes, create_settings(ins, op)); - auto solutions = problem.GetSolutions(ctx.get_current_device().get_gfx_name()); - tc.solutions.resize(solutions.size()); - std::iota(tc.solutions.begin(), tc.solutions.end(), 0); - std::vector gemm_shapes{shapes[0], shapes[1], shapes.back()}; - tc.problem = to_value(gemm_shapes); - return tc; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/ck_gemm_softmax_gemm.cpp b/docker/rocm/migraphx/targets/gpu/jit/ck_gemm_softmax_gemm.cpp deleted file mode 100644 index 693153d09..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/ck_gemm_softmax_gemm.cpp +++ /dev/null @@ -1,236 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const ck_gemm_softmax_gemm_kernel = R"__migraphx__( -#include -#include -#include -#include -#include -#include -#include <${include}> - -namespace migraphx { - -${preamble} - -extern "C" { - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - auto settings = make_ck_gemm_softmax_gemm_settings(MIGRAPHX_MAKE_CONSTANT(float{SCALE})); - ck_gemm_softmax_gemm<${solution}, ${blocks_per_batch}>(settings, xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct ck_gemm_softmax_gemm_compiler : compiler -{ - std::vector names() const - { - return {"ck_gemm_softmax_gemm", "gpu::ck_gemm_softmax_gemm"}; - } - - ck::host::device_batched_gemm_softmax_gemm::Problem - create_problem(const std::vector& inputs, const value&) const - { - const auto& a_shape = inputs[0]; - const auto& b_shape = inputs[1]; - const auto& b1_shape = inputs[2]; - const auto& c_shape = inputs.back(); - - // cppcheck-suppress unreadVariable - auto rank = a_shape.ndim(); - auto batch_count = get_batch_count(c_shape); - auto m = c_shape.lens()[rank - 2]; - m = can_fold_batch(inputs) ? m * batch_count : m; - auto n = c_shape.lens().back(); - auto k = a_shape.lens().back(); - auto o = c_shape.lens().back(); - - const bool trans_a = transposed_matrix(a_shape); - const bool trans_b = transposed_matrix(b_shape); - const bool trans_b1 = transposed_matrix(b1_shape); - const bool trans_c = transposed_matrix(c_shape); - const auto a_type = get_type(a_shape); - const auto b_type = get_type(b_shape); - const auto b1_type = get_type(b1_shape); - const auto c_type = get_type(c_shape); - - std::string ck_passthrough = "ck_passthrough"; - return ck::host::device_batched_gemm_softmax_gemm::Problem{m, - n, - k, - o, - trans_a, - trans_b, - trans_b1, - trans_c, - a_type, - b_type, - b1_type, - c_type, - ck_passthrough, - ck_passthrough, - ck_passthrough, - ck_passthrough}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - const auto& c_shape = inputs.back(); - auto tuning_value = v.get("tuning_value", 5); - auto batch_count = get_batch_count(c_shape); - auto problem = create_problem(inputs, v); - - const auto include_header = problem.GetIncludeHeader(); - const auto solutions = problem.GetSolutions(ctx.get_current_device().get_gfx_name()); - const auto& solution = solutions.at(tuning_value); - const auto template_str = solution.template_str; - const auto blocks_per_batch = solution.grid_size; - const auto block_size = solution.block_size; - - hip_compile_options options; - options.additional_src_files = ck_headers(); - auto grid_size = can_fold_batch(inputs) ? blocks_per_batch : batch_count * blocks_per_batch; - options.set_launch_params(v, grid_size * block_size, block_size); - options.inputs = inputs; - options.output = c_shape; - options.kernel_name = v.get("kernel", "ck_gemm_softmax_gemm_kernel"); - options.virtual_inputs = inputs; - if(can_fold_batch(inputs)) - { - auto vinputs = inputs; - fold_batch_dims(vinputs[0]); - remove_batch_dims(vinputs[1]); - std::for_each(vinputs.begin() + 2, vinputs.end(), fold_batch_dims); - options.virtual_inputs = vinputs; - } - - if(v.get("check", false) or enabled(MIGRAPHX_CK_DEBUG{})) - options.emplace_param("-DMIGRAPHX_CK_CHECK=1"); - - // scale - assert(v.contains("scale")); - auto scale = v.at("scale").to(); - options.emplace_param("-DSCALE=" + std::to_string(scale)); - - auto src = interpolate_string(ck_gemm_softmax_gemm_kernel, - {{"solution", template_str}, - {"include", include_header}, - {"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"blocks_per_batch", to_string(blocks_per_batch)}, - {"preamble", v.get("preamble", std::string{})}, - {"kernel", options.kernel_name}}); - - return compile_hip_code_object(ctx, src, options); - } - - value create_settings(instruction_ref ins, const operation& op) const - { - auto v = op.to_value(); - v["kernel"] = "ck_gemm_softmax_gemm_kernel"; - if(not ins->module_inputs().empty()) - { - auto* pm = ins->module_inputs().front(); - v["preamble"] = generate_pointwise(*pm, "post_ck_gemm_softmax_gemm_function") + - "\nMIGRAPHX_LIFT_CLASS(post_ck_gemm_softmax_gemm, " - "post_ck_gemm_softmax_gemm_function);"; - v["post"] = "ck_function_adaptor"; - v["kernel"] = "ck_gemm_softmax_gemm_" + generate_name_from_ops(*pm) + "_kernel"; - } - return v; - } - - compiler_replace - compile(context& ctx, instruction_ref ins, const operation& op, const value& solution) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = create_settings(ins, op); - if(not solution.is_null()) - v["tuning_value"] = solution; - return {compile_op(ctx, shapes, v), - [=](module& m, instruction_ref ins2, const operation& code_object) { - if(enabled(MIGRAPHX_LOG_CK_GEMM{})) - { - std::vector gemm_shapes{ - shapes[0], shapes[1], shapes.back().with_type(shapes[0].type())}; - std::cout << "gpu::ck_gemm_softmax_gemm: " - << to_json_string(to_value(gemm_shapes)) << std::endl; - } - m.replace_instruction(ins2, code_object, ins2->inputs()); - }}; - } - - optional - get_tuning_config(context& ctx, instruction_ref ins, const operation& op, bool exhaustive) const - { - if(not exhaustive and not enabled(MIGRAPHX_TUNE_CK{})) - return nullopt; - tuning_config tc; - auto shapes = to_shapes(ins->inputs()); - auto problem = create_problem(shapes, create_settings(ins, op)); - auto solutions = problem.GetSolutions(ctx.get_current_device().get_gfx_name()); - tc.solutions.resize(solutions.size()); - std::iota(tc.solutions.begin(), tc.solutions.end(), 0); - std::vector gemm_shapes{shapes[0], shapes[1], shapes.back()}; - tc.problem = to_value(gemm_shapes); - return tc; - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/compute_attention_probabilities.cpp b/docker/rocm/migraphx/targets/gpu/jit/compute_attention_probabilities.cpp deleted file mode 100644 index 8a0c72207..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/compute_attention_probabilities.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const compute_attention_probabilities_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - - compute_attention_probabilities(xs..., make_gqa_parameters(${gqa_params})); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct compute_attention_probabilities_compiler : compiler -{ - std::vector names() const - { - return {"compute_attention_probabilities", "gpu::compute_attention_probabilities"}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto params = init_params(inputs, v); - auto gqa_params_str = params.make_init_str(); - - hip_compile_options options; - options.set_launch_params( - v, - compute_global_for(ctx, - params.batch_size * params.num_heads * params.sequence_length * - params.seqlen_present_kv_cache)); - options.inputs = inputs; - options.output = inputs.back(); - options.kernel_name = v.get("kernel", "compute_attention_probabilities_kernel"); - - auto src = interpolate_string(compute_attention_probabilities_kernel, - {{"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"gqa_params", gqa_params_str}, - {"kernel", options.kernel_name}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = op.to_value(); - return compile_op(ctx, shapes, v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/compute_attention_scores.cpp b/docker/rocm/migraphx/targets/gpu/jit/compute_attention_scores.cpp deleted file mode 100644 index a8834a24e..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/compute_attention_scores.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const compute_attention_scores_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - - -namespace migraphx { - - - -extern "C" { - - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - - compute_attention_scores(xs..., make_gqa_parameters(${gqa_params})); - }); -} - - -} - -} // namespace migraphx - -)__migraphx__"; - -struct compute_attention_scores_compiler : compiler -{ - std::vector names() const - { - return {"compute_attention_scores", "gpu::compute_attention_scores"}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto params = init_params(inputs, v); - auto gqa_params_str = params.make_init_str(); - - hip_compile_options options; - options.set_launch_params( - v, - compute_global_for(ctx, - params.batch_size * params.num_heads * params.sequence_length * - params.head_size)); - options.inputs = inputs; - options.output = inputs.back(); - options.kernel_name = v.get("kernel", "compute_attention_scores_kernel"); - - auto src = interpolate_string(compute_attention_scores_kernel, - {{"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"gqa_params", gqa_params_str}, - {"kernel", options.kernel_name}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = op.to_value(); - return compile_op(ctx, shapes, v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/concat.cpp b/docker/rocm/migraphx/targets/gpu/jit/concat.cpp deleted file mode 100644 index 322f863e3..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/concat.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const concat_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -${preamble} - -extern "C" { - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last(), ${transformers})(${args})([](auto y, ${concat_params}, auto... xs) { - concat<${axis}>(${concat_args})(${post}, y, xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct concat_compiler : compiler -{ - std::vector names() const { return {"fused_concat", "concat"}; } - - static std::vector normalize(std::vector inputs, std::size_t& axis) - { - auto s = inputs.back(); - std::vector strides(s.lens().size()); - strides[axis] = 1; - - inputs.push_back(shape{s.type(), s.lens(), strides}); - - auto result = reduce_dims(normalize_permutation(inputs)); - auto rstrides = result.back().strides(); - auto it = std::find_if(rstrides.begin(), rstrides.end(), [](auto x) { return x == 1; }); - axis = it - rstrides.begin(); - result.pop_back(); - return result; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - options.inputs = inputs; - options.output = inputs.back(); - auto concat_axis = v.at("axis").to(); - options.virtual_inputs = normalize(inputs, concat_axis); - options.kernel_name = v.get("kernel", "concat_kernel"); - auto axis = find_fast_axis(options.virtual_inputs); - auto op_names = v.at("ops").to_vector(); - auto args = v.at("args"); - vectorize vec{}; - if(axis != concat_axis) - vec = vectorize::elements(ctx, axis, options.virtual_inputs); - auto nelements_per_op = options.virtual_inputs.back().elements() / op_names.size(); - options.set_launch_params(v, compute_global_for(ctx, nelements_per_op / vec.size, 256)); - options.emplace_param("-Wno-float-equal"); - std::vector concat_params; - std::vector concat_args; - for(auto i : range(op_names.size())) - { - const auto& name = op_names[i]; - auto n = args.at(name).to(); - auto prefix = to_c_id(name + std::to_string(i) + "_concat_x"); - transform(range(n), std::back_inserter(concat_params), [&](auto j) { - return "auto " + prefix + std::to_string(j); - }); - std::vector pack_args = {"MIGRAPHX_LIFT(" + name + ")"}; - transform(range(n), std::back_inserter(pack_args), [&](auto j) { - return prefix + std::to_string(j); - }); - concat_args.push_back("pack(" + join_strings(pack_args, ", ") + ")"); - } - auto src = interpolate_string(concat_kernel, - {{"kernel", options.kernel_name}, - {"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"concat_params", join_strings(concat_params, ", ")}, - {"concat_args", join_strings(concat_args, ", ")}, - {"post", v.get("post", std::string{"op::id{}"})}, - {"transformers", make_transformer_args(vec)}, - {"preamble", v.get("preamble", std::string{})}, - {"axis", std::to_string(concat_axis)}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto v = op.to_value(); - if(op.name() == "fused_concat") - { - std::unordered_map mod_names_lookup; - transform(range(ins->module_inputs().size()), - std::inserter(mod_names_lookup, mod_names_lookup.end()), - [&](auto i) { - return std::make_pair(ins->module_inputs()[i]->name(), - "pointwise" + std::to_string(i)); - }); - v["preamble"] = transform_accumulate( - ins->module_inputs().begin(), - ins->module_inputs().end(), - std::string{}, - std::plus<>{}, - [&](module_ref mod) { - return generate_pointwise(*mod, mod_names_lookup.at(mod->name())) + "\n"; - }); - std::vector mod_names; - std::transform(ins->module_inputs().begin(), - ins->module_inputs().end() - 1, - std::back_inserter(mod_names), - [&](module_ref mod) { return mod_names_lookup.at(mod->name()); }); - v["ops"] = mod_names; - module_ref last_mod = ins->module_inputs().back(); - v["post"] = "MIGRAPHX_LIFT(" + mod_names_lookup.at(last_mod->name()) + ")"; - std::unordered_map mod_args; - std::transform(ins->module_inputs().begin(), - ins->module_inputs().end() - 1, - std::inserter(mod_args, mod_args.end()), - [&](module_ref mod) { - const auto& name = mod_names_lookup.at(mod->name()); - return std::make_pair(name, mod->get_parameter_names().size()); - }); - v["args"] = mod_args; - auto prefix_name = transform_accumulate(ins->module_inputs().begin(), - ins->module_inputs().end() - 1, - std::string{}, - std::plus<>{}, - [&](module_ref mod) -> std::string { - auto name = generate_name_from_ops(*mod); - if(name.empty()) - return ""; - return name + "_"; - }); - v["kernel"] = prefix_name + "concat_" + - generate_name_from_ops(*(ins->module_inputs().back())) + "_kernel"; - } - else if(op.name() == "concat") - { - auto concat_inputs = ins->inputs().size() - 1; - if(not ins->module_inputs().empty()) - { - auto* pm = ins->module_inputs().front(); - concat_inputs = ins->inputs().size() - pm->get_parameter_names().size(); - v["preamble"] = generate_pointwise(*pm, "post_concat"); - v["post"] = "MIGRAPHX_LIFT(post_concat)"; - v["kernel"] = "concat_" + generate_name_from_ops(*pm) + "_kernel"; - } - std::vector mod_names(concat_inputs, "op::id{}"); - v["ops"] = mod_names; - std::unordered_map mod_args = {{"op::id{}", 1}}; - v["args"] = mod_args; - } - return compile_op(ctx, to_shapes(ins->inputs()), v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/concat_past_present.cpp b/docker/rocm/migraphx/targets/gpu/jit/concat_past_present.cpp deleted file mode 100644 index b18d70108..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/concat_past_present.cpp +++ /dev/null @@ -1,119 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const concat_past_present_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - - - -extern "C" { - - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors())(${args})([](auto... xs) { - - concat_past_present(xs..., make_gqa_parameters(${gqa_params})); - }); -} - - -} - -} // namespace migraphx - -)__migraphx__"; - -struct concat_past_present_compiler : compiler -{ - std::vector names() const - { - return {"concat_past_present", "gpu::concat_past_present"}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto params = init_params(inputs, v); - auto gqa_params_str = params.make_init_str(); - - hip_compile_options options; - options.set_launch_params(v, - compute_global_for(ctx, - 2 * params.batch_size * params.kv_num_heads * - params.sequence_length * - params.head_size)); - options.inputs = inputs; - options.output = inputs.front(); - options.kernel_name = v.get("kernel", "concat_past_present_kernel"); - options.output_arg = 0; - - auto src = interpolate_string(concat_past_present_kernel, - {{"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"gqa_params", gqa_params_str}, - {"kernel", options.kernel_name}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = op.to_value(); - return compile_op(ctx, shapes, v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/gather.cpp b/docker/rocm/migraphx/targets/gpu/jit/gather.cpp deleted file mode 100644 index 9dc17db09..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/gather.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// NOLINTNEXTLINE -static const char* const gather_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void gather_kernel(void* in_data, void* in_indices, void* output) -{ - make_tensors()(in_data, in_indices, output)([](auto&&... xs) { - gather<${axis}>(xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct gather_compiler : compiler -{ - std::vector names() const { return {"gather"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - const auto& out_s = inputs.back(); - options.set_launch_params(v, compute_global_for(ctx, out_s.elements())); - options.inputs = inputs; - options.output = out_s; - options.kernel_name = "gather_kernel"; - options.virtual_inputs = inputs; - - auto axis = v.at("axis").to(); - - auto src = interpolate_string(gather_kernel, {{"axis", axis}}); - - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/gathernd.cpp b/docker/rocm/migraphx/targets/gpu/jit/gathernd.cpp deleted file mode 100644 index 05a48f4e9..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/gathernd.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// NOLINTNEXTLINE -static const char* const gathernd_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void gathernd_kernel(void* in_data, void* in_indices, void* output) -{ - make_tensors()(in_data, in_indices, output)([](auto&&... xs) { - auto settings = make_gathernd_settings(MIGRAPHX_MAKE_CONSTANT(int64_t{BATCH_DIMS})); - gathernd(xs..., settings); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct gathernd_compiler : compiler -{ - std::vector names() const { return {"gathernd"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - const auto& out_s = inputs.back(); - options.set_launch_params(v, compute_global_for(ctx, out_s.elements())); - options.inputs = inputs; - options.output = out_s; - options.kernel_name = "gathernd_kernel"; - options.virtual_inputs = inputs; - - // batch_dims - assert(v.contains("batch_dims")); - auto batch_dims = v.at("batch_dims").to(); - options.emplace_param("-DBATCH_DIMS=" + std::to_string(batch_dims)); - - return compile_hip_code_object(ctx, gathernd_kernel, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/gqa_rotary_embedding.cpp b/docker/rocm/migraphx/targets/gpu/jit/gqa_rotary_embedding.cpp deleted file mode 100644 index 340616352..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/gqa_rotary_embedding.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const gqa_rotary_embedding_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - - - -extern "C" { - - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - - gqa_rotary_embedding(xs..., make_gqa_parameters(${gqa_params})); - }); -} - - -} - -} // namespace migraphx - -)__migraphx__"; - -struct gqa_rotary_embedding_compiler : compiler -{ - std::vector names() const - { - return {"gqa_rotary_embedding", "gpu::gqa_rotary_embedding"}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto params = init_params(inputs, v); - auto gqa_params_str = params.make_init_str(); - - hip_compile_options options; - options.set_launch_params(v, compute_global_for(ctx, inputs.back().elements())); - options.inputs = inputs; - options.output = inputs.back(); - options.kernel_name = v.get("kernel", "gqa_rotary_embedding_kernel"); - - auto src = interpolate_string(gqa_rotary_embedding_kernel, - {{"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"gqa_params", gqa_params_str}, - {"kernel", options.kernel_name}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = op.to_value(); - return compile_op(ctx, shapes, v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/gqa_softmax.cpp b/docker/rocm/migraphx/targets/gpu/jit/gqa_softmax.cpp deleted file mode 100644 index c1ff241dd..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/gqa_softmax.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -// NOLINTNEXTLINE -static const char* const gqa_softmax_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - - - -extern "C" { - - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - - gqa_softmax(xs..., make_gqa_parameters(${gqa_params})); - }); -} - - -} - -} // namespace migraphx - -)__migraphx__"; - -struct gqa_softmax_compiler : compiler -{ - std::vector names() const { return {"gqa_softmax", "gpu::gqa_softmax"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto params = init_params(inputs, v); - auto gqa_params_str = params.make_init_str(); - - hip_compile_options options; - options.set_launch_params( - v, - compute_global_for(ctx, params.batch_size * params.num_heads * params.sequence_length)); - options.inputs = inputs; - options.output = inputs.back(); - options.kernel_name = v.get("kernel", "gqa_softmax_kernel"); - - auto src = interpolate_string(gqa_softmax_kernel, - {{"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"gqa_params", gqa_params_str}, - {"kernel", options.kernel_name}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto shapes = to_shapes(ins->inputs()); - auto v = op.to_value(); - return compile_op(ctx, shapes, v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/layernorm.cpp b/docker/rocm/migraphx/targets/gpu/jit/layernorm.cpp deleted file mode 100644 index 09736031b..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/layernorm.cpp +++ /dev/null @@ -1,131 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -static const char* const layernorm_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - -${preamble} - -extern "C" { -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last(), ${transformers})(${args})([](auto... xs) { - ${layernorm}<${axis}>(${post}, ${eps}, xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct layernorm_compiler : compiler -{ - std::vector names() const - { - return {"layernorm", "gpu::prelayernorm", "gpu::preadd_layernorm"}; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - // TODO: Use reduce_dims - auto axis = inputs.front().lens().size() - 1; - auto faxis = find_fast_axis({inputs.front()}); - vectorize vec{}; - // Vectorize if the axis is a reduction axis - if(axis == faxis) - { - vec = vectorize::elements(ctx, faxis, inputs); - } - auto relements = inputs[0].lens()[axis] / vec.size; - auto nelements = (inputs.back().elements() / inputs[0].lens()[axis]); - auto block_size = compute_block_size(ctx, relements, 256); - hip_compile_options options; - options.set_launch_params( - v, compute_global_for(ctx, nelements * block_size, 256), block_size); - options.output = inputs.back(); - options.inputs = inputs; - options.kernel_name = v.get("kernel", "layernorm_kernel"); - auto eps = v.get("epsilon", 1e-12f); - - auto src = interpolate_string(layernorm_kernel, - {{"kernel", options.kernel_name}, - {"params", enum_params(inputs.size(), "void * private_p")}, - {"args", enum_params(inputs.size(), "private_p")}, - {"transformers", make_transformer_args(vec)}, - {"post", v.get("post", std::string{"op::id{}"})}, - {"preamble", v.get("preamble", std::string{})}, - {"layernorm", v.get("layernorm", std::string{"layernorm"})}, - {"axis", to_string(axis)}, - {"eps", to_string(eps)}}); - - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - auto v = op.to_value(); - v["layernorm"] = "layernorm"; - v["kernel"] = "layernorm_kernel"; - if(op.name() == "gpu::preadd_layernorm") - { - v["layernorm"] = "add_layernorm"; - v["kernel"] = "add_layernorm_kernel"; - } - if(not ins->module_inputs().empty()) - { - auto* pm = ins->module_inputs().front(); - v["preamble"] = generate_pointwise(*pm, "post_layernorm"); - v["post"] = "MIGRAPHX_LIFT(post_layernorm)"; - v["kernel"] = - v["layernorm"].to() + "_" + generate_name_from_ops(*pm) + "_kernel"; - } - return compile_op(ctx, to_shapes(ins->inputs()), v); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/mlir.cpp b/docker/rocm/migraphx/targets/gpu/jit/mlir.cpp deleted file mode 100644 index 4893743c2..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/mlir.cpp +++ /dev/null @@ -1,283 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_DUMP_TO_MXR); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_DUMP); - -static module create_pointwise_module(module_ref in_mod) -{ - module pw_mod; - std::unordered_map map_ins; - for(auto param : in_mod->get_parameters()) - { - map_ins[param] = - pw_mod.add_parameter(any_cast(param->get_operator()).parameter, - shape{param->get_shape().type()}); - } - auto return_args = pw_mod.add_instructions( - in_mod, - &map_ins, - [](module& m, - instruction_ref ins, - const operation& op, - const std::vector& inputs, - const std::vector& mod_args) -> instruction_ref { - if(op.name() == "multibroadcast" and inputs.front()->name() == "@literal") - return inputs.front(); - else - return m.insert_instruction(ins, op, inputs, mod_args); - }); - pw_mod.add_return(return_args); - return pw_mod; -} - -struct mlir_compiler : compiler -{ - std::vector names() const { return {"gpu::mlir_op"}; } - - operation compile_op(context&, const std::vector&, const value&) const { return {}; } - - compiler_replace - compile(context& ctx, instruction_ref ins, const operation&, const value& solution) const - { - auto* smod = ins->module_inputs().front(); - assert(smod->get_parameter_names().size() == ins->inputs().size() - 1); - auto gemm_like_ins = std::find_if(smod->begin(), smod->end(), [&](const auto& i) { - return contains({"dot", "quant_dot", "convolution", "quant_convolution"}, i.name()); - }); - auto pointwise_ins = std::find_if(gemm_like_ins, smod->end(), [&](const auto& i) { - return i.get_operator().attributes().get("pointwise", false) == true; - }); - - // check if (a) module is fused (b) contains a "gemm/conv" instruction and (c) - // perfConfig can not allow fused module - if(gemm_like_ins != smod->end() and pointwise_ins != smod->end() and - not is_module_fusible(*smod, ctx, solution)) - { - auto input_args = ins->inputs(); - // remove alloc buffer - input_args.pop_back(); - auto split_ins = std::prev(pointwise_ins); - std::array mod_splits; - mod_splits = smod->split(input_args, {split_ins}); - auto dot_mlir_inputs = to_shapes(mod_splits[0].inputs); - // add alloc for the gemm output - dot_mlir_inputs.push_back(mod_splits[0].mod.get_output_shapes().front()); - mlir_code_object cop1 = compile_mlir(ctx, mod_splits[0].mod, dot_mlir_inputs, solution); - auto pw_shapes = to_shapes(mod_splits[1].inputs); - if(mod_splits[1].mod.get_output_shapes().size() == 1) - { - pw_shapes.push_back(mod_splits[1].mod.get_output_shapes().front()); - } - else - { - pw_shapes.push_back(shape{mod_splits[1].mod.get_output_shapes()}); - } - assert(pw_shapes.back() == ins->get_shape()); - auto pw_mod = create_pointwise_module(&mod_splits[1].mod); - auto cop2 = compile_pointwise(ctx, pw_shapes, &pw_mod); - std::vector cops = {cop1, - mlir_code_object{any_cast(cop2)}}; - return insert(cops, mod_splits, ins, split_ins); - } - return insert(compile_mlir(ctx, *smod, to_shapes(ins->inputs()), solution)); - } - - compiler_replace insert(const mlir_code_object& mco) const - { - return {std::vector{mco.cop}, - [=](module& m, instruction_ref ins, const std::vector& ops) { - std::vector inputs = ins->inputs(); - - // Tuple inputs not supported - assert(std::all_of(inputs.begin(), inputs.end() - 1, [](auto i) { - return i->get_shape().sub_shapes().empty(); - })); - - // Multiple output case (allocate ins will give a tuple) - std::vector flat_inputs(inputs); - bool multi_out = not flat_inputs.back()->get_shape().sub_shapes().empty(); - if(multi_out) - { - auto allocs = flat_inputs.back(); - flat_inputs.pop_back(); - auto sub_shape_idx = range(allocs->get_shape().sub_shapes().size()); - std::transform(sub_shape_idx.begin(), - sub_shape_idx.end(), - std::back_inserter(flat_inputs), - [&](int i) { - return m.insert_instruction( - ins, - migraphx::make_op("get_tuple_elem", {{"index", i}}), - allocs); - }); - } - std::vector tuple_replacements; - - for(const auto i : range(mco.prefill_indices.size())) - { - auto prefilled_ins = m.insert_instruction( - ins, - migraphx::make_op("hip::fill", {{"value", mco.prefill_values[i]}}), - flat_inputs[mco.prefill_indices[i]]); - if(not multi_out or mco.prefill_indices[i] < inputs.size() - 1) - { - replace(inputs, inputs[mco.prefill_indices[i]], prefilled_ins); - } - else - { - tuple_replacements.push_back(prefilled_ins); - } - } - - if(multi_out and not tuple_replacements.empty()) - { - // Add identity to make sure fill operations happen before kernel call - tuple_replacements.insert(tuple_replacements.begin(), inputs.back()); - inputs.back() = m.insert_instruction( - ins, migraphx::make_op("identity"), tuple_replacements); - } - - auto mlir = insert_mlir(m, ins, any_cast(ops.front()), inputs); - return m.replace_instruction(ins, mlir); - }, - &trace}; - } - - compiler_replace insert(const std::vector& mcos, - const std::array& mods, - instruction_ref precompile_ins, - instruction_ref split_ins) const - { - std::vector cobjs(mcos.size()); - std::transform( - mcos.begin(), mcos.end(), cobjs.begin(), [](const auto& mco) { return mco.cop; }); - auto precompiled_inputs = precompile_ins->inputs(); - return { - cobjs, [=](module& m, instruction_ref ins, const std::vector& ops) { - auto compiled_inputs = ins->inputs(); - std::unordered_map inputs_rep_map; - for(const auto i : range(precompiled_inputs.size())) - { - inputs_rep_map[precompiled_inputs[i]] = compiled_inputs[i]; - } - auto dot_inputs = mods[0].inputs; - auto dot_mod_out_shape = mods[0].mod.get_output_shapes().front(); - auto dot_alloc = m.insert_instruction( - ins, - migraphx::make_op("hip::allocate", {{"shape", to_value(dot_mod_out_shape)}})); - dot_inputs.push_back(dot_alloc); - for(const auto i : range(mcos[0].prefill_indices.size())) - { - auto prefilled_ins = m.insert_instruction( - ins, - migraphx::make_op("hip::fill", {{"value", mcos[0].prefill_values[i]}}), - dot_inputs[mcos[0].prefill_indices[i]]); - replace(dot_inputs, dot_inputs[mcos[0].prefill_indices[i]], prefilled_ins); - } - - std::vector dot_inputs_updated; - std::transform(dot_inputs.begin(), - dot_inputs.end(), - std::back_inserter(dot_inputs_updated), - [&](const auto& i) { - if(inputs_rep_map.find(i) != inputs_rep_map.end()) - { - assert(inputs_rep_map.at(i)->get_shape() == i->get_shape()); - return inputs_rep_map.at(i); - } - return i; - }); - auto mlir_ins = - insert_mlir(m, ins, any_cast(ops[0]), dot_inputs_updated); - auto pwm = mods[1]; - pwm.replace(split_ins, mlir_ins); - auto pw_inputs = pwm.inputs; - pw_inputs.push_back(ins->inputs().back()); - std::vector pw_inputs_updated; - std::transform(pw_inputs.begin(), - pw_inputs.end(), - std::back_inserter(pw_inputs_updated), - [&](const auto& i) { - if(inputs_rep_map.find(i) != inputs_rep_map.end()) - { - assert(inputs_rep_map.at(i)->get_shape() == i->get_shape()); - return inputs_rep_map.at(i); - } - return i; - }); - auto pw_ins = - insert_mlir(m, ins, any_cast(ops[1]), pw_inputs_updated); - return m.replace_instruction(ins, pw_ins); - }}; - } - - optional get_tuning_config(const context& ctx, - instruction_ref ins, - const operation&, - bool exhaustive) const - { - static const auto mxr_loc = string_value_of(MIGRAPHX_MLIR_DUMP_TO_MXR{}); - static const auto mlir_loc = string_value_of(MIGRAPHX_MLIR_DUMP{}); - - auto shapes = to_shapes(ins->inputs()); - auto* smod = ins->module_inputs().front(); - if(not mxr_loc.empty()) - { - dump_mlir_to_mxr(*smod, ins->inputs(), mxr_loc); - } - if(not mlir_loc.empty()) - { - dump_mlir_to_file(*smod, shapes, mlir_loc); - } - return get_tuning_config_mlir(ctx, *smod, shapes, exhaustive); - } - - static void trace(std::ostream& os, instruction_ref ins) - { - auto shapes = to_shapes(ins->inputs()); - auto* smod = ins->module_inputs().front(); - os << dump_mlir(*smod, shapes); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/pad.cpp b/docker/rocm/migraphx/targets/gpu/jit/pad.cpp deleted file mode 100644 index 9ea77ee99..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/pad.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -static const char* const pointwise_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { -MIGRAPHX_GLOBAL void pad_kernel(void* input_p, void* output_p) -{ - auto offsets = index_ints<${offsets}>{}; - auto idx = make_index(); - make_tensors()(input_p, output_p)([&](auto input, auto output) { - pad(idx, offsets, input, output, ${pad_val}); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct pad_compiler : compiler -{ - std::vector names() const { return {"pad"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto padding = v.at("pads").to_vector(); - auto input_lens = inputs.front().lens(); - std::vector offsets(input_lens.size()); - std::copy(padding.begin(), padding.begin() + offsets.size(), offsets.begin()); - - auto offset_lens = input_lens; - std::transform(input_lens.begin(), - input_lens.end(), - offsets.begin(), - offset_lens.begin(), - [&](auto input, auto offset) { return input + offset; }); - - auto vinputs = inputs; - vinputs.push_back(inputs.front().with_lens(offset_lens)); - auto rinputs = reduce_dims(normalize_permutation(vinputs)); - - auto rinput_lens = rinputs.front().lens(); - auto roffset_lens = rinputs.back().lens(); - std::vector roffsets(roffset_lens.size()); - std::transform(rinput_lens.begin(), - rinput_lens.end(), - roffset_lens.begin(), - roffsets.begin(), - [](auto input, auto offset_dim) { return offset_dim - input; }); - rinputs.pop_back(); - - hip_compile_options options; - options.inputs = inputs; - options.output = inputs.back(); - options.virtual_inputs = rinputs; - options.kernel_name = "pad_kernel"; - options.set_launch_params(v, compute_global_for(ctx, inputs.at(1).elements())); - - auto pad_val = v.get("value", 0.f); - auto pad_val_string = to_string(pad_val); - if(float_equal(pad_val, std::numeric_limits::lowest())) - pad_val_string = "lowest{}"; - if(float_equal(pad_val, std::numeric_limits::max())) - pad_val_string = "highest{}"; - - auto src = interpolate_string( - pointwise_kernel, - {{"pad_val", to_string(pad_val_string)}, {"offsets", to_string_range(roffsets)}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/pointwise.cpp b/docker/rocm/migraphx/targets/gpu/jit/pointwise.cpp deleted file mode 100644 index 9e352888e..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/pointwise.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -static const char* const pointwise_kernel = R"__migraphx__( -#include -#include -#include - -namespace migraphx { - -${preamble} - -extern "C" { -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - auto idx = make_index(); - pointwise<${noutputs}, ${tiled}>(idx, ${transformers})(${lambda}, ${args}); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct pointwise_compiler : compiler -{ - std::vector names() const { return {"pointwise", "contiguous", "layout"}; } - - static std::size_t oversubscribe_if(bool b) - { - if(b) - return 256; - else - return 1; - } - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - options.inputs = flatten(inputs); - options.output = inputs.back(); - options.virtual_inputs = reduce_dims(normalize_permutation(options.inputs)); - options.emplace_param("-Wno-float-equal"); - auto axis = find_fast_axis(options.virtual_inputs); - auto vec = vectorize::elements(ctx, axis, options.virtual_inputs); - options.kernel_name = v.get("kernel", "kernel"); - auto noutputs = options.inputs.size() - inputs.size() + 1; - auto t = tile::elements(options.virtual_inputs, noutputs); - // auto t = tile{}; - if(t.ntiles == 0) - options.set_launch_params( - v, compute_global_for(ctx, options.inputs.front().elements() / vec.size, 256)); - else - options.set_launch_params( - v, compute_global_for(ctx, t.ntiles * t.block_size, 256), t.block_size); - auto src = - interpolate_string(pointwise_kernel, - {{"kernel", options.kernel_name}, - {"params", enum_params(options.inputs.size(), "void * private_p")}, - {"args", enum_params(options.inputs.size(), "private_p")}, - {"lambda", v.at("lambda").to()}, - {"transformers", make_transformer_args(t, vec)}, - {"tiled", t.ntiles > 0 ? "true" : "false"}, - {"noutputs", std::to_string(noutputs)}, - {"preamble", v.get("preamble", std::string{})}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - if(contains({"layout", "contiguous"}, op.name())) - { - return compile_op(ctx, - to_shapes(ins->inputs()), - {{"lambda", "[](auto x) { return make_tuple(x); }"}, - {"kernel", op.name() + "_kernel"}}); - } - else - { - assert(not ins->module_inputs().empty()); - const_module_ref pm = ins->module_inputs().front(); - return compile_pointwise(ctx, to_shapes(ins->inputs()), pm); - } - } -}; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/pooling.cpp b/docker/rocm/migraphx/targets/gpu/jit/pooling.cpp deleted file mode 100644 index f245a2269..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/pooling.cpp +++ /dev/null @@ -1,193 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// NOLINTNEXTLINE -static const char* const pooling_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void pooling_kernel(void* in_data, void* output) -{ - transform_args(make_tensors(), rotate_last())(in_data, output)([](auto&&... xs) { - pooling<${algo}, ${group_size}>(${op}, make_window(index_ints<${window}>{}, index_ints<${stride}>{}, index_ints<${padding}>{}), xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct pooling_compiler : compiler -{ - - static std::size_t compute_subwave_size(context& ctx, std::size_t n) - { - std::size_t max_wavefront_size = ctx.get_current_device().get_wavefront_size(); - std::size_t wavefront_size = 1; - while(wavefront_size <= n and wavefront_size < max_wavefront_size) - wavefront_size *= 2; - return wavefront_size / 2; - } - - struct algorithm - { - std::string name = "reduce::lane"; - std::size_t reduce_size = 1; - std::size_t block_size = 256; - std::size_t group_size = 1; - - static std::size_t compute_group_size(const shape& output) - { - auto n = output.lens().back(); - const std::size_t max_group_size = 32; - std::size_t group_size = 1; - while((n % (group_size * 2) == 0) and group_size <= max_group_size) - group_size *= 2; - return group_size; - } - - algorithm() {} - - algorithm(context& ctx, const shape& input, const std::vector& window) - { - if(input.strides().back() != 1) - return; - std::size_t max_wavefront_size = ctx.get_current_device().get_wavefront_size(); - auto wsize = window.back(); - if(wsize > max_wavefront_size) - { - block_size = compute_block_size(ctx, wsize, 256); - reduce_size = block_size; - name = "reduce::block"; - } - else - { - block_size = max_wavefront_size; - reduce_size = compute_subwave_size(ctx, wsize); - name = "reduce::subwave<" + to_string(reduce_size) + ">"; - } - } - }; - - template - static void normalize(std::vector& inputs, Ts&... xs) - { - auto perm = find_permutation(inputs); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](auto s) { - return reorder_shape(s, perm); - }); - each_args([&](auto& dims) { dims = reorder_dims(dims, perm); }, xs...); - } - - std::vector names() const { return {"pooling"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - const auto& out_s = inputs.back(); - options.inputs = inputs; - options.output = out_s; - options.kernel_name = "pooling_kernel"; - options.virtual_inputs = inputs; - - auto ndim = out_s.ndim(); - auto pool_ndim = ndim - 2; - - auto read_value = [&](const std::string& name, std::size_t def) { - if(v.contains(name)) - { - std::vector result(2, def); - auto x = v.at(name).to_vector(); - if(x.size() >= pool_ndim) - result.insert(result.end(), x.begin(), x.begin() + pool_ndim); - return result; - } - else - { - std::vector result(ndim, def); - return result; - } - }; - - auto padding = read_value("padding", 0); - auto stride = read_value("stride", 1); - auto window = read_value("lengths", 1); - - const auto& mode_v = v.at("mode"); - std::string mode = - mode_v.is_string() ? mode_v.get_string() : to_string(mode_v.to()); - bool count_include_pad = v.get("count_include_pad", false); - if(count_include_pad and mode == "average") - mode = "average_include_pad"; - - std::string op = mode + "_pool"; - if(mode == "lpnorm") - op += "<" + v.at("lp_order").to() + ">"; - - algorithm algo{}; - options.set_launch_params( - v, - compute_global_for(ctx, (out_s.elements() / algo.group_size) * algo.reduce_size, 256), - algo.block_size); - normalize(options.virtual_inputs, padding, stride, window); - auto src = interpolate_string(pooling_kernel, - {{"op", op + "{}"}, - {"algo", algo.name}, - {"group_size", to_string(algo.group_size)}, - {"window", to_string_range(window)}, - {"stride", to_string_range(stride)}, - {"padding", to_string_range(padding)}}); - - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/reduce.cpp b/docker/rocm/migraphx/targets/gpu/jit/reduce.cpp deleted file mode 100644 index bdf7313f5..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/reduce.cpp +++ /dev/null @@ -1,408 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -static const char* const simple_reduce_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -${preamble} - -extern "C" { -MIGRAPHX_GLOBAL void reduce_kernel(void* input_p, void* output_p) -{ - - transform_args(make_tensors(), ${transformers})(input_p, output_p)([](auto input, auto output) { - - simple_reduce(${reduction}, ${init}, input, output, ${read}, ${write}); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -static std::vector get_reduce_lens(const std::vector& input_lens, - const std::vector& output_lens) -{ - std::vector reduce_lens; - std::transform(output_lens.begin(), - output_lens.end(), - input_lens.begin(), - std::back_inserter(reduce_lens), - [](auto x, auto y) -> std::size_t { - if(x == y) - return 1; - else - return y; - }); - return reduce_lens; -} - -template -static shape get_reduced_shape(const shape& s, const std::vector& axes) -{ - auto lens = s.lens(); - std::fill(lens.begin(), lens.end(), 1); - for(const auto& axis : axes) - lens[axis] = s.lens()[axis]; - return s.with_lens(lens); -} - -template -static shape get_output_shape(const shape& s, const std::vector& axes) -{ - auto lens = s.lens(); - for(const auto& axis : axes) - lens[axis] = 1; - return s.with_lens(lens); -} - -template -static std::string get_reduce_algo(context& ctx, const std::vector& inputs, ReduceLens rlens) -{ - const auto init = std::numeric_limits::max(); - auto relements = std::accumulate(rlens.begin(), rlens.end(), 1, std::multiplies<>{}); - // The minimum stride - auto min_stride = std::inner_product( - rlens.begin(), - rlens.end(), - inputs.front().strides().begin(), - init, - [](auto x, auto y) { return std::min(x, y); }, - [](auto len, auto stride) { return len == 1 ? init : stride; }); - if(min_stride > 2) - return "lane"; - if(relements <= ctx.get_current_device().get_wavefront_size()) - return "wave"; - return "block"; -} - -static std::string get_reduce_algo(context& ctx, const std::vector& inputs) -{ - auto rlens = get_reduce_lens(inputs.front().lens(), inputs.back().lens()); - return get_reduce_algo(ctx, inputs, rlens); -} - -static std::size_t compute_subwave_size(context& ctx, std::size_t n) -{ - std::size_t max_wavefront_size = ctx.get_current_device().get_wavefront_size(); - std::size_t wavefront_size = 1; - while(wavefront_size <= n and wavefront_size < max_wavefront_size) - wavefront_size *= 2; - return wavefront_size; -} - -/// This will adjust the input shapes so a partial reduction is done per workgroup. -/// This is done by splitting the reduction axis so each split group becomes -/// part of the batch. So if we want to do a split redution of a tensor -/// {K}, then this will create a tensor of {K/N, N} where N is the number of -/// split groups. To compute the number of split groups it finds the largest -/// divisor that can divide K to make it less than min_size. -static std::vector split_reduce(const std::vector& inputs, - std::size_t min_size = 1024) -{ - std::vector result; - auto input_shape = inputs.front(); - const auto& reduce_shape = inputs[inputs.size() - 2]; - const auto& output_shape = inputs[inputs.size() - 1]; - - auto is = range(reduce_shape.lens().size()); - using array_type = std::array; - auto initial = array_type{std::numeric_limits::max(), - std::numeric_limits::max()}; - auto faxis = transform_accumulate( - is.begin(), is.end(), initial, MIGRAPHX_LIFT(std::min), [&](auto i) -> array_type { - if(input_shape.lens()[i] == output_shape.lens()[i]) - return initial; - return {input_shape.strides()[i], std::size_t(i)}; - })[1]; - - assert(faxis < reduce_shape.lens().size()); - - std::size_t n = 1; - auto r = input_shape.lens()[faxis]; - auto factors = make_array(2, 3, 5, 7, 11); - while(r > min_size) - { - // NOLINTNEXTLINE(readability-qualified-auto) - auto it = std::find_if(factors.begin(), factors.end(), [&](auto d) { return r % d == 0; }); - if(it == factors.end()) - break; - r /= *it; - n *= *it; - } - assert(n != 1); - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(result), [&](const shape& s) -> shape { - auto lens = s.lens(); - auto strides = s.strides(); - - lens.push_back(n); - if(lens[faxis] == 1) - { - strides.push_back(0); - } - else - { - lens[faxis] /= n; - strides.push_back(strides[faxis] * lens[faxis]); - } - - return {s.type(), lens, strides}; - }); - return reduce_dims(normalize_permutation(result)); -} - -struct simple_reduce_compiler : compiler -{ - std::vector names() const - { - return {"simple_reduce", - "reduce_sum", - "reduce_mean", - "reduce_max", - "reduce_min", - "reduce_prod", - "reduce_any", - "reduce_all"}; - } - - static std::size_t get_reduce_elements(const std::vector& inputs) - { - return inputs.front().elements() / inputs.back().elements(); - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - options.inputs = inputs; - options.output = inputs.back(); - options.virtual_inputs = reduce_dims(inputs); - auto faxis = find_fast_axis({options.virtual_inputs.front()}); - vectorize vec{}; - auto nelements = options.virtual_inputs.back().elements(); - auto algo = v.get("algo", get_reduce_algo(ctx, options.virtual_inputs)); - if(algo == "block" or algo == "wave") - { - // Vectorize if the axis is a reduction axis - if(options.virtual_inputs.back().lens()[faxis] == 1) - vec = vectorize::elements(ctx, faxis, options.virtual_inputs); - auto relements = get_reduce_elements(options.virtual_inputs) / vec.size; - if(algo == "block") - { - auto block_size = compute_block_size(ctx, relements, 256); - if(relements >= block_size * 256) - algo = "block_large"; - options.set_launch_params( - v, compute_global_for(ctx, nelements * block_size, 256), block_size); - } - else - { - auto subwave_size = compute_subwave_size(ctx, relements); - algo = "subwave<" + std::to_string(subwave_size) + ">"; - options.set_launch_params(v, - compute_global_for(ctx, nelements * subwave_size, 256), - ctx.get_current_device().get_wavefront_size()); - } - } - else if(algo == "lane") - { - options.set_launch_params(v, compute_global_for(ctx, nelements, 256)); - } - else - { - MIGRAPHX_THROW("Unknown reduce algo: " + algo); - } - options.kernel_name = "reduce_kernel"; - std::string identity = "[](auto x) { return x; }"; - auto src = interpolate_string(simple_reduce_kernel, - {{"reduction", v.at("reduction").to()}, - {"init", v.get("init", std::string{"0"})}, - {"read", v.get("read", identity)}, - {"write", v.get("write", identity)}, - {"algo", algo}, - {"transformers", make_transformer_args(vec)}, - {"preamble", v.get("preamble", std::string{})}}); - options.emplace_param("-Wno-float-equal"); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - value v = value::object{}; - reduce_op r{}; - r.set(ins, op); - v["reduction"] = r.reduction; - v["read"] = r.read; - v["write"] = r.write; - v["init"] = r.init; - return compile_op(ctx, to_shapes(ins->inputs()), v); - } -}; - -static const char* const fused_reduce_kernel = R"__migraphx__( -#include -#include -#include -#include -#include - -namespace migraphx { - -${preamble} - -extern "C" { -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), ${transformers}, rotate_and_pack_last<${noutputs}>())(${args})([](auto y, auto... xs) { - fused_reduce(y, ${assign}{}, partial(${lambda})(xs...)); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct fused_reduce_compiler : compiler -{ - std::vector names() const { return {"fused_reduce", "split_fused_reduce"}; } - - static shape get_input_shape(const std::vector& inputs) - { - auto it = std::max_element(inputs.begin(), - inputs.end(), - by(std::less<>{}, [](const shape& s) { return s.elements(); })); - return *it; - } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - auto assign = v.get("assign", "assign_none"); - auto axes = v.at("axes").to_vector(); - auto finputs = flatten(inputs); - auto noutputs = finputs.size() - inputs.size() + 1; - auto virtual_inputs = finputs; - virtual_inputs.push_back(get_reduced_shape(get_input_shape(finputs), axes)); - virtual_inputs.push_back(get_output_shape(get_input_shape(finputs), axes)); - virtual_inputs = reduce_dims(normalize_permutation(virtual_inputs)); - if(assign != "assign_none") - virtual_inputs = split_reduce(virtual_inputs); - auto reduce_output_shape = virtual_inputs.back(); - virtual_inputs.pop_back(); - auto reduction_shape = virtual_inputs.back(); - virtual_inputs.pop_back(); - - hip_compile_options options; - options.inputs = finputs; - options.output = inputs.back(); - options.virtual_inputs = virtual_inputs; - auto faxis = find_fast_axis({options.virtual_inputs.front()}); - vectorize vec{}; - auto nelements = reduce_output_shape.elements(); - auto algo = - v.get("algo", get_reduce_algo(ctx, options.virtual_inputs, reduction_shape.lens())); - if(algo == "block" or algo == "wave") - { - // Vectorize if the axis is a reduction axis - if(reduce_output_shape.lens()[faxis] == 1) - vec = vectorize::elements(ctx, faxis, options.virtual_inputs); - auto relements = reduction_shape.elements() / vec.size; - if(algo == "block") - { - auto block_size = compute_block_size(ctx, relements, 256); - if(relements >= block_size * 256) - algo = "block_large"; - options.set_launch_params( - v, compute_global_for(ctx, nelements * block_size, 256), block_size); - } - else - { - auto subwave_size = compute_subwave_size(ctx, relements); - algo = "subwave<" + std::to_string(subwave_size) + ">"; - options.set_launch_params(v, - compute_global_for(ctx, nelements * subwave_size, 256), - ctx.get_current_device().get_wavefront_size()); - } - } - else if(algo == "lane") - { - options.set_launch_params(v, compute_global_for(ctx, nelements, 256)); - } - else - { - MIGRAPHX_THROW("Unknown reduce algo: " + algo); - } - options.kernel_name = v.get("kernel", "reduce_kernel"); - auto src = interpolate_string( - fused_reduce_kernel, - {{"kernel", options.kernel_name}, - {"params", enum_params(finputs.size(), "void * private_p")}, - {"args", enum_params(finputs.size(), "private_p")}, - {"assign", assign}, - {"algo", algo}, - {"reduced", "decltype(" + generate_make_shape(reduce_output_shape) + ")"}, - {"lambda", v.at("lambda").to()}, - {"transformers", make_transformer_args(vec)}, - {"noutputs", std::to_string(noutputs)}, - {"preamble", v.get("preamble", std::string{})}}); - options.emplace_param("-Wno-float-equal"); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - assert(not ins->module_inputs().empty()); - auto v = op.to_value(); - auto* rm = ins->module_inputs().front(); - v["preamble"] = generate_reduce(*rm, "fused_reduce_op"); - v["lambda"] = "MIGRAPHX_LIFT(fused_reduce_op)"; - v["kernel"] = generate_name_from_ops(*rm) + "_kernel"; - return compile_op(ctx, to_shapes(ins->inputs()), v); - } -}; -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/roialign.cpp b/docker/rocm/migraphx/targets/gpu/jit/roialign.cpp deleted file mode 100644 index aeaf7a858..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/roialign.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#if !MIGRAPHX_USE_MIOPEN -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// NOLINTNEXTLINE -static const char* const roialign_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void roialign_kernel(void* in_x, void* in_rois, void* in_ind, void* y) -{ - make_tensors()(in_x, in_rois, in_ind, y)([](auto&&... xs) { - auto settings = make_roalign_settings(MIGRAPHX_MAKE_CONSTANT(float{ROIS_OFFSET}), - _c, - _c, - MIGRAPHX_MAKE_CONSTANT(float{SPATIAL_SCALE})); - roialign(xs..., settings); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct roialign_compiler : compiler -{ - std::vector names() const { return {"roialign"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - options.set_launch_params(v, compute_global_for(ctx, inputs.back().elements()), 128); - options.output = inputs.back(); - options.inputs = inputs; - options.kernel_name = "roialign_kernel"; - - // sampling_ratio - options.emplace_param("-DSAMPLING_RATIO=" + v.at("sampling_ratio").to()); - - // pooling_mode - auto mode = v.at("mode").to(); - std::string is_avg_pooling = - (mode == migraphx::op::pooling_mode::average) ? "true" : "false"; - options.emplace_param("-DIS_AVG_POOLING=" + is_avg_pooling); - - // coord_trans_mode - auto ctm = v.at("coordinate_transformation_mode").to(); - float rois_offset = (ctm == "half_pixel") ? -0.5f : 0.0f; - options.emplace_param("-DROIS_OFFSET=" + std::to_string(rois_offset)); - - // spatial_scale - options.emplace_param("-DSPATIAL_SCALE=" + v.at("spatial_scale").to()); - - return compile_hip_code_object(ctx, roialign_kernel, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/scatter.cpp b/docker/rocm/migraphx/targets/gpu/jit/scatter.cpp deleted file mode 100644 index 1a1264f23..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/scatter.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "scatter.hpp" - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// NOLINTNEXTLINE -static const char* const scatter_elements_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void scatter_elements_kernel(void* in_indices, void* in_updates, void* output) -{ - make_tensors()(in_indices, in_updates, output)([](auto&&... xs) { - scatter<${axis}, ${skip_out_of_bounds}>(xs..., ${reduction}{}); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct scatter_elements_compiler : scatter_compiler -{ - std::vector names() const - { - return {"scatter_none", "scatter_add", "scatter_mul", "scatter_min", "scatter_max"}; - } - - std::string make_interpolated_string(const operation& op) const - { - const auto reduction = op.name().substr(std::char_traits::length("scatter_")); - auto axis = std::to_string(op.to_value().get("axis", 0)); - auto skip_out_of_bounds = std::to_string(op.to_value().get("skip_out_of_bounds", 0)); - - return interpolate_string(scatter_elements_kernel, - {{"reduction", "assign_" + reduction}, - {"axis", axis}, - {"skip_out_of_bounds", skip_out_of_bounds}}); - } - - std::string get_kernel_name(const operation&) const { return "scatter_elements_kernel"; } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/scatter.hpp b/docker/rocm/migraphx/targets/gpu/jit/scatter.hpp deleted file mode 100644 index 6fb955647..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/scatter.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_JIT_SCATTER_HPP -#define MIGRAPHX_GUARD_JIT_SCATTER_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -template -struct scatter_compiler : compiler -{ - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - const auto inputs = - to_shapes(std::vector{ins->inputs().begin() + 1, ins->inputs().end()}); - - hip_compile_options options; - options.set_launch_params(op.to_value(), compute_global_for(ctx, inputs.at(1).elements())); - options.inputs = inputs; - options.output = inputs.back(); - options.kernel_name = derived().get_kernel_name(op); - options.virtual_inputs = inputs; - options.emplace_param("-DMIGRAPHX_ALLOW_ATOMIC_CAS=1"); - - const auto src = derived().make_interpolated_string(op); - return prepend_copy_data_to_output(compile_hip_code_object(ctx, src, options)); - } - - // ONNX spec states the following for ScatterElements and ScatterND: - // "The output of the operation is produced by creating a copy of the input data, ..." - // The sole responsibility of the MIGraphX Scatter operator implementations being to perform the - // update operations as specified by ONNX, it is necessary to place the copying of the input - // data before the MIGraphX operator in the graph. - compiler_replace prepend_copy_data_to_output(const operation& co) const - { - return {co, [](module& m, instruction_ref ins, const operation& op) { - auto args = ins->inputs(); - args.back() = - m.insert_instruction(ins, make_op("hip::copy"), args.front(), args.back()); - args.erase(args.begin()); - return m.replace_instruction(ins, op, args); - }}; - } - - std::string get_kernel_name(const operation& op) const { return op.name() + "_kernel"; } - - const Derived& derived() const { return static_cast(*this); } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/jit/scatternd.cpp b/docker/rocm/migraphx/targets/gpu/jit/scatternd.cpp deleted file mode 100644 index cf3ab00ed..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/scatternd.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "scatter.hpp" - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -// NOLINTNEXTLINE -static const char* const scatternd_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void scatternd_kernel(void* in_indices, void* in_updates, void* output) -{ - make_tensors()(in_indices, in_updates, output)([](auto&&... xs) { - scatternd(xs..., ${reduction}{}); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct scatternd_compiler : scatter_compiler -{ - std::vector names() const - { - return { - "scatternd_none", "scatternd_add", "scatternd_mul", "scatternd_min", "scatternd_max"}; - } - - std::string make_interpolated_string(const operation& op) const - { - const auto reduction = op.name().substr(std::char_traits::length("scatternd_")); - return interpolate_string(scatternd_kernel, {{"reduction", "assign_" + reduction}}); - } - - std::string get_kernel_name(const operation&) const { return "scatternd_kernel"; } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/softmax.cpp b/docker/rocm/migraphx/targets/gpu/jit/softmax.cpp deleted file mode 100644 index d2e24a233..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/softmax.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_USE_FAST_SOFTMAX) - -using namespace migraphx::gpu::gen; // NOLINT - -static const char* const softmax_kernel = R"__migraphx__( -#include -#include -#include -#include - -namespace migraphx { - -extern "C" { -MIGRAPHX_GLOBAL void softmax_kernel(void* input_p, void* output_p) -{ - transform_args(make_tensors(), ${transformers})(input_p, output_p)([](auto input, auto output) { - softmax<${axis}>(input, output); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct softmax_compiler : compiler -{ - std::vector names() const { return {"softmax"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - // TODO: Use reduce_dims - auto axis = v.at("axis").to(); - auto faxis = find_fast_axis({inputs.front()}); - vectorize vec{}; - // Vectorize if the axis is a reduction axis - if(faxis == axis) - { - vec = vectorize::elements(ctx, faxis, inputs); - } - auto relements = inputs[0].lens()[axis] / vec.size; - auto nelements = (inputs.back().elements() / inputs[0].lens()[axis]); - auto block_size = compute_block_size(ctx, relements, 256); - hip_compile_options options; - options.set_launch_params( - v, compute_global_for(ctx, nelements * block_size, 256), block_size); - options.output = inputs.back(); - options.inputs = inputs; - options.kernel_name = "softmax_kernel"; - - if(enabled(MIGRAPHX_USE_FAST_SOFTMAX{})) - options.emplace_param("-DMIGRAPHX_USE_FAST_SOFTMAX"); - - auto src = interpolate_string( - softmax_kernel, - {{"transformers", make_transformer_args(vec)}, {"axis", to_string(axis)}}); - - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/jit/unpack_int4.cpp b/docker/rocm/migraphx/targets/gpu/jit/unpack_int4.cpp deleted file mode 100644 index 68f2038b8..000000000 --- a/docker/rocm/migraphx/targets/gpu/jit/unpack_int4.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include "migraphx/instruction.hpp" -#include "migraphx/instruction_ref.hpp" -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -using namespace migraphx::gpu::gen; // NOLINT - -static const char* const unpack_int4_kernel = R"__migraphx__( -#include -#include - -namespace migraphx { - -extern "C" { - -MIGRAPHX_GLOBAL void ${kernel}(${params}) -{ - transform_args(make_tensors(), rotate_last())(${args})([](auto... xs) { - unpack_int4<${axis}>(xs...); - }); -} - -} - -} // namespace migraphx - -)__migraphx__"; - -struct unpack_int4_compiler : compiler -{ - std::vector names() const { return {"unpack_int4"}; } - - operation compile_op(context& ctx, const std::vector& inputs, const value& v) const - { - hip_compile_options options; - options.inputs = inputs; - options.output = inputs.back(); - options.virtual_inputs = reduce_dims(normalize_permutation(options.inputs)); - options.kernel_name = "unpack_int4_kernel"; - options.set_launch_params(v, compute_global_for(ctx, inputs.front().elements())); - - auto src = - interpolate_string(unpack_int4_kernel, - {{"kernel", options.kernel_name}, - {"params", enum_params(options.inputs.size(), "void * private_p")}, - {"args", enum_params(options.inputs.size(), "private_p")}, - {"axis", std::to_string(v.at("axis").to())}}); - return compile_hip_code_object(ctx, src, options); - } - - compiler_replace compile(context& ctx, instruction_ref ins, const operation& op) const - { - return compile_op(ctx, to_shapes(ins->inputs()), op.to_value()); - } -}; - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/kernel.cpp b/docker/rocm/migraphx/targets/gpu/kernel.cpp deleted file mode 100644 index a7c79bded..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernel.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#else -// extern declare the function since hip/hip_ext.h header is broken -extern hipError_t hipExtModuleLaunchKernel(hipFunction_t, // NOLINT - uint32_t, - uint32_t, - uint32_t, - uint32_t, - uint32_t, - uint32_t, - size_t, - hipStream_t, - void**, - void**, - hipEvent_t = nullptr, - hipEvent_t = nullptr, - uint32_t = 0); -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -extern std::string hip_error(int error); - -using hip_module_ptr = MIGRAPHX_MANAGE_PTR(hipModule_t, hipModuleUnload); - -struct kernel_impl -{ - hip_module_ptr module = nullptr; - hipFunction_t fun = nullptr; -}; - -hip_module_ptr load_module(const char* image) -{ - hipModule_t raw_m; - auto status = hipModuleLoadData(&raw_m, image); - hip_module_ptr m{raw_m}; - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to load module: " + hip_error(status)); - return m; -} - -kernel::kernel(const char* image, const std::string& name) : impl(std::make_shared()) -{ - impl->module = load_module(image); - auto status = hipModuleGetFunction(&impl->fun, impl->module.get(), name.c_str()); - if(hipSuccess != status) - MIGRAPHX_THROW("Failed to get function: " + name + ": " + hip_error(status)); -} - -void launch_kernel(hipFunction_t fun, - hipStream_t stream, - std::size_t global, - std::size_t local, - void* kernargs, - std::size_t size, - hipEvent_t start, - hipEvent_t stop) -{ - assert(global > 0); - assert(local > 0); - void* config[] = { -// HIP_LAUNCH_PARAM_* are macros that do horrible things -#ifdef MIGRAPHX_USE_CLANG_TIDY - nullptr, kernargs, nullptr, &size, nullptr -#else - HIP_LAUNCH_PARAM_BUFFER_POINTER, - kernargs, - HIP_LAUNCH_PARAM_BUFFER_SIZE, - &size, - HIP_LAUNCH_PARAM_END -#endif - }; - - auto status = hipExtModuleLaunchKernel(fun, - global, - 1, - 1, - local, - 1, - 1, - 0, - stream, - nullptr, - reinterpret_cast(&config), - start, - stop); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to launch kernel: " + hip_error(status)); - if(stop != nullptr) - { - status = hipEventSynchronize(stop); - if(status != hipSuccess) - MIGRAPHX_THROW("Failed to sync event: " + hip_error(status)); - } -} - -void kernel::launch(hipStream_t stream, - std::size_t global, - std::size_t local, - std::vector args, - hipEvent_t start, - hipEvent_t stop) const -{ - assert(impl != nullptr); - void* kernargs = reinterpret_cast(args.data()); - std::size_t size = args.size() * sizeof(void*); - - launch_kernel(impl->fun, stream, global, local, kernargs, size, start, stop); -} - -void kernel::launch(hipStream_t stream, - std::size_t global, - std::size_t local, - const std::vector& args, - hipEvent_t start, - hipEvent_t stop) const -{ - assert(impl != nullptr); - std::vector kernargs = pack_args(args); - std::size_t size = kernargs.size(); - - launch_kernel(impl->fun, stream, global, local, kernargs.data(), size, start, stop); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/algorithm.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/algorithm.hpp deleted file mode 100644 index 2e5b376c2..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/algorithm.hpp +++ /dev/null @@ -1,334 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_ALGORITHM_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_ALGORITHM_HPP - -#include - -namespace migraphx { - -template -constexpr void swap(T& a, T& b) noexcept -{ - T old = a; - a = b; - b = old; -} - -template -constexpr void iter_swap(Iterator1 a, Iterator2 b) -{ - if(a == b) - return; - swap(*a, *b); -} - -struct less -{ - template - constexpr auto operator()(T x, U y) const - { - return x < y; - } -}; - -struct greater -{ - template - constexpr auto operator()(T x, U y) const - { - return x > y; - } -}; - -template -constexpr T accumulate(InputIt first, InputIt last, T init, BinaryOperation op) -{ - for(; first != last; ++first) - { - init = op(static_cast(init), *first); - } - return init; -} - -template -constexpr OutputIt copy(InputIt first, InputIt last, OutputIt d_first) -{ - while(first != last) - { - *d_first++ = *first++; - } - return d_first; -} - -template -constexpr OutputIt copy_if(InputIt first, InputIt last, OutputIt d_first, UnaryPredicate pred) -{ - for(; first != last; ++first) - { - if(pred(*first)) - { - *d_first = *first; - ++d_first; - } - } - return d_first; -} - -template -constexpr Iterator is_sorted_until(Iterator first, Iterator last, Compare comp) -{ - if(first != last) - { - Iterator next = first; - while(++next != last) - { - if(comp(*next, *first)) - return next; - first = next; - } - } - return last; -} - -template -constexpr bool is_sorted(Iterator first, Iterator last, Compare comp) -{ - return is_sorted_until(first, last, comp) == last; -} - -template -constexpr F for_each(Iterator first, Iterator last, F f) -{ - for(; first != last; ++first) - { - f(*first); - } - return f; -} - -template -constexpr Iterator find_if(Iterator first, Iterator last, Predicate p) -{ - for(; first != last; ++first) - { - if(p(*first)) - { - return first; - } - } - return last; -} - -template -constexpr Iterator find(Iterator first, Iterator last, const T& value) -{ - return find_if(first, last, [&](const auto& x) { return x == value; }); -} - -template -constexpr bool any_of(InputIt first, InputIt last, UnaryPredicate p) -{ - return find_if(first, last, p) != last; -} - -template -constexpr bool none_of(InputIt first, InputIt last, UnaryPredicate p) -{ - return find_if(first, last, p) == last; -} - -template -constexpr bool all_of(InputIt first, InputIt last, UnaryPredicate p) -{ - return none_of(first, last, [=](auto&& x) { return not p(x); }); -} - -template -constexpr Iterator1 search(Iterator1 first, Iterator1 last, Iterator2 s_first, Iterator2 s_last) -{ - for(;; ++first) - { - Iterator1 it = first; - for(Iterator2 s_it = s_first;; ++it, ++s_it) - { - if(s_it == s_last) - { - return first; - } - if(it == last) - { - return last; - } - if(not(*it == *s_it)) - { - break; - } - } - } -} - -template -constexpr T inner_product(InputIt1 first1, - InputIt1 last1, - InputIt2 first2, - T init, - BinaryOperation1 op1, - BinaryOperation2 op2) -{ - while(first1 != last1) - { - init = op1(init, op2(*first1, *first2)); - ++first1; - ++first2; - } - return init; -} - -template -constexpr T inner_product(InputIt1 first1, InputIt1 last1, InputIt2 first2, T init) -{ - return inner_product( - first1, - last1, - first2, - init, - [](auto x, auto y) { return x + y; }, - [](auto x, auto y) { return x * y; }); -} - -template -constexpr bool equal(Iterator1 first1, Iterator1 last1, Iterator2 first2, BinaryPred p) -{ - for(; first1 != last1; ++first1, ++first2) - if(not p(*first1, *first2)) - { - return false; - } - return true; -} - -template -constexpr void iota(Iterator first, Iterator last, T value) -{ - for(; first != last; ++first, ++value) - *first = value; -} - -template -constexpr Iterator min_element(Iterator first, Iterator last, Compare comp) -{ - if(first == last) - return last; - - Iterator smallest = first; - - while(++first != last) - if(comp(*first, *smallest)) - smallest = first; - - return smallest; -} - -template -constexpr Iterator rotate(Iterator first, Iterator middle, Iterator last) -{ - if(first == middle) - return last; - - if(middle == last) - return first; - - Iterator write = first; - Iterator next_read = first; - - for(Iterator read = middle; read != last; ++write, ++read) - { - if(write == next_read) - next_read = read; - iter_swap(write, read); - } - - rotate(write, next_read, last); - return write; -} - -template -constexpr Iterator upper_bound(Iterator first, Iterator last, const T& value, Compare comp) -{ - auto count = last - first; - - while(count > 0) - { - auto it = first; - auto step = count / 2; - it += step; - - if(not comp(value, *it)) - { - first = ++it; - count -= step + 1; - } - else - count = step; - } - - return first; -} - -template -constexpr void sort(Iterator first, Iterator last, Compare comp) -{ - if(first == last) - return; - for(auto i = first; i != last - 1; ++i) - iter_swap(i, min_element(i, last, comp)); - MIGRAPHX_ASSERT(is_sorted(first, last, comp)); -} - -template -constexpr void sort(Iterator first, Iterator last) -{ - sort(first, last, less{}); -} - -template -constexpr void stable_sort(Iterator first, Iterator last, Compare comp) -{ - if(first == last) - return; - for(auto i = first; i != last; ++i) - rotate(upper_bound(first, i, *i, comp), i, i + 1); - MIGRAPHX_ASSERT(is_sorted(first, last, comp)); -} - -template -constexpr void stable_sort(Iterator first, Iterator last) -{ - stable_sort(first, last, less{}); -} - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/args.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/args.hpp deleted file mode 100644 index 2706e14a4..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/args.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_ARGS_HPP -#define MIGRAPHX_GUARD_KERNELS_ARGS_HPP - -#include -#include - -namespace migraphx { - -// Use template specialization since ADL is broken on hcc -template -struct make_tensor; - -template -__device__ auto make_tensors_impl(F f, detail::seq, Ts*... xs) -{ - return f(make_tensor::apply(xs)...); -} - -inline __device__ auto make_tensors() -{ - return [](auto*... xs) { - return [=](auto f) { return make_tensors_impl(f, detail::gens{}, xs...); }; - }; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_ARGS_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/array.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/array.hpp deleted file mode 100644 index 623d4d8dd..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/array.hpp +++ /dev/null @@ -1,388 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_ARRAY_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_ARRAY_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_ARRAY_OP(op, binary_op) \ - template \ - constexpr array& operator op(const array& x) \ - { \ - array_detail::array_for_each(*this, x)([](auto& sy, auto sx) { sy op sx; }); \ - return *this; \ - } \ - template {})> \ - constexpr array& operator op(const U& x) \ - { \ - array_detail::array_for_each (*this)([&](auto& sy) { sy op x; }); \ - return *this; \ - } \ - template \ - friend constexpr auto operator binary_op(const array& x, const array& y) \ - { \ - array z{}; \ - array_detail::array_for_each(z, x, y)( \ - [&](auto& sz, auto sx, auto sy) { sz = sx binary_op sy; }); \ - return z; \ - } \ - template {})> \ - friend constexpr auto operator binary_op(const array& x, const U& y) \ - { \ - array z{}; \ - array_detail::array_for_each(z, x)([&](auto& sz, auto sx) { sz = sx binary_op y; }); \ - return z; \ - } \ - template {})> \ - friend constexpr auto operator binary_op(const U& x, const array& y) \ - { \ - array z{}; \ - array_detail::array_for_each(z, y)([&](auto& sz, auto sy) { sz = x binary_op sy; }); \ - return z; \ - } - -namespace array_detail { -template -constexpr auto is_vectorizable() -{ - return not is_same{} and (is_fundamental{} or is_same{}); -} - -template -__device__ auto& array2vec(T& x) -{ - using value_type = typename T::value_type; - constexpr auto size = decltype(x.size()){}; - using type = vec; - if constexpr(is_const{}) - return reinterpret_cast(x); - else - return reinterpret_cast(x); -} - -template -constexpr auto array_for_each(T& x, Ts&... xs) -{ - MIGRAPHX_ASSERT(((x.size() == xs.size()) and ...)); - return [&](auto f) { - constexpr auto size = decltype(x.size()){}; - if constexpr((is_vectorizable() or - (is_vectorizable() or ...)) and - size <= 8 and size > 1 and (size % 2 == 0)) - { - if(__builtin_is_constant_evaluated()) - { - for(index_int i = 0; i < size; i++) - f(x[i], xs[i]...); - } - else - { - using vec_type = remove_reference_t; - f(array2vec(x), __builtin_convertvector(array2vec(xs), vec_type)...); - } - } - else - { - for(index_int i = 0; i < size; i++) - f(x[i], xs[i]...); - } - }; -} -} // namespace array_detail - -template -struct array -{ - using value_type = T; - T d[N]; - - constexpr array() = default; - - template {} and ...))> - constexpr array(Ts... xs) : d{xs...} - { - } - - template {} and (N > 1))> - constexpr explicit array(U x) - { - for(index_int i = 0; i < N; i++) - d[i] = x; - } - - constexpr T& operator[](index_int i) - { - MIGRAPHX_ASSERT(i < N); - return d[i]; - } - constexpr const T& operator[](index_int i) const - { - MIGRAPHX_ASSERT(i < N); - return d[i]; - } - - constexpr T& front() { return d[0]; } - constexpr const T& front() const { return d[0]; } - - constexpr T& back() { return d[N - 1]; } - constexpr const T& back() const { return d[N - 1]; } - - constexpr T* data() { return d; } - constexpr const T* data() const { return d; } - - constexpr index_constant size() const { return {}; } - constexpr auto empty() const { return size() == _c<0>; } - - constexpr T* begin() { return d; } - constexpr const T* begin() const { return d; } - - constexpr T* end() { return d + size(); } - constexpr const T* end() const { return d + size(); } - - constexpr T dot(const array& x) const - { - auto r = x * (*this); - return r.reduce([](auto a, auto b) { return a + b; }, 0); - } - - constexpr T product() const - { - return reduce([](auto x, auto y) { return x * y; }, 1); - } - - constexpr T single(index_int width = 100) const - { - T result = 0; - T a = 1; - for(index_int i = 0; i < N; i++) - { - result += d[N - i - 1] * a; - a *= width; - } - return result; - } - - template - constexpr auto apply(F f) const - { - array result; - for(index_int i = 0; i < N; i++) - result[i] = f(d[i]); - return result; - } - - template - constexpr auto reduce(F f, T init) const - { - T result = init; - for(index_int i = 0; i < N; i++) - result = f(result, d[i]); - return result; - } - - MIGRAPHX_DEVICE_ARRAY_OP(+=, +) - MIGRAPHX_DEVICE_ARRAY_OP(-=, -) - MIGRAPHX_DEVICE_ARRAY_OP(*=, *) - MIGRAPHX_DEVICE_ARRAY_OP(/=, /) - MIGRAPHX_DEVICE_ARRAY_OP(%=, %) - MIGRAPHX_DEVICE_ARRAY_OP(&=, &) - MIGRAPHX_DEVICE_ARRAY_OP(|=, |) - MIGRAPHX_DEVICE_ARRAY_OP(^=, ^) - - friend constexpr bool operator==(const array& x, const array& y) - { - for(index_int i = 0; i < N; i++) - { - if(x[i] != y[i]) - return false; - } - return true; - } - - template {})> - friend constexpr bool operator==(const array& x, const U& y) - { - for(index_int i = 0; i < N; i++) - { - if(x[i] != y) - return false; - } - return true; - } - - template {})> - friend constexpr bool operator==(const U& x, const array& y) - { - return y == x; - } - - template - friend constexpr bool operator!=(const U& x, const array& y) - { - return not(x == y); - } - template - friend constexpr bool operator!=(const array& x, const U& y) - { - return not(x == y); - } - // This uses the product order rather than lexical order - friend constexpr bool operator<(const array& x, const array& y) - { - for(index_int i = 0; i < N; i++) - { - if(not(x[i] < y[i])) - return false; - } - return true; - } - friend constexpr bool operator>(const array& x, const array& y) { return y < x; } - friend constexpr bool operator<=(const array& x, const array& y) { return (x < y) or (x == y); } - friend constexpr bool operator>=(const array& x, const array& y) { return (y < x) or (x == y); } - - constexpr array carry(array result) const - { - index_int overflow = 0; - for(diff_int i = result.size() - 1; i > 0; i--) - { - auto z = result[i] + overflow; - // Reset overflow - overflow = 0; - // Compute overflow using while loop instead of mod - while(z >= d[i]) - { - z -= d[i]; - overflow += 1; - } - result[i] = z; - } - result[0] += overflow; - return result; - } - - /// Get the multi-dimensional index from the given 1D index. - constexpr array multi(T idx) const - { - array result; - index_int tidx = idx; - for(diff_int is = result.size() - 1; is > 0; is--) - { - result[is] = tidx % d[is]; - tidx = tidx / d[is]; - } - result[0] = tidx; - return result; - } - - template - friend constexpr const Stream& operator<<(const Stream& ss, const array& a) - { - for(index_int i = 0; i < N; i++) - { - if(i > 0) - ss << ", "; - ss << a[i]; - } - return ss; - } -}; - -template -constexpr auto array_apply(F f) -{ - return [=](auto&& x) { return x.apply(f); }; -} - -template -constexpr array make_array(T x, Ts... xs) -{ - return {x, static_cast(xs)...}; -} -template -struct integral_const_array : array -{ - using base_array = array; - MIGRAPHX_DEVICE_CONSTEXPR integral_const_array() : base_array({Xs...}) {} - - constexpr const base_array& base() const { return *this; } -}; - -template -constexpr auto make_const_array(T x, Ts... xs) -{ - return integral_const_array{}; -} - -template -constexpr auto generate_array(N n, F f) -{ - return sequence_c([=](auto... is) { return array{f(is)...}; }); -} - -template -constexpr auto unpack(integral_const_array, F f) -{ - return f(_c...); -} - -template -constexpr auto transform(integral_const_array, F f) -{ - return integral_const_array{}; -} - -template -constexpr auto transform_i(integral_const_array, F f) -{ - return sequence_c( - [=](auto... is) { return integral_const_array{}; }); -} - -template -constexpr auto transform(integral_const_array, integral_const_array, F f) -{ - return integral_const_array{}; -} - -template -constexpr auto return_array_c(F f) -{ - constexpr auto r = f(); - return sequence(r.size(), [&](auto... is) { return make_const_array(_c...); }); -} - -template -using index_ints = integral_const_array; - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/atomic.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/atomic.hpp deleted file mode 100644 index 76e0409cc..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/atomic.hpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_ATOMIC_HPP -#define MIGRAPHX_GUARD_KERNELS_ATOMIC_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef MIGRAPHX_ALLOW_ATOMIC_CAS -// NOLINTNEXTLINE -#define MIGRAPHX_ALLOW_ATOMIC_CAS 0 -#endif - -// NOLINTNEXTLINE -#define MIGRAPHX_ATOMIC_CAS_WARNING() \ - MIGRAPHX_ASSERT(MIGRAPHX_ALLOW_ATOMIC_CAS and "Using atomicCAS is slow") - -namespace migraphx { -namespace atomic { - -using cas_rank = rank<1>; - -template -MIGRAPHX_DEVICE_CONSTEXPR void cas(rank<1>, T& x, T y, Op op) -{ - MIGRAPHX_ATOMIC_CAS_WARNING(); - using storage = conditional_t; - storage* address = reinterpret_cast(&x); - storage expected = __hip_atomic_load(address, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - while(not __hip_atomic_compare_exchange_strong(address, - &expected, - bit_cast(op(bit_cast(expected), y)), - __ATOMIC_RELAXED, - __ATOMIC_RELAXED, - __HIP_MEMORY_SCOPE_AGENT)) - { - } -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto cas(rank<0>, vec& x, vec y, Op op) - -> decltype(cas(cas_rank{}, x[0], y[0], op), void()) -{ - for(index_int i = 0; i < N; i++) - { - cas(cas_rank{}, x[i], y[i], op); - } -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto builtin_assign(T& x, T y, op::sum) - MIGRAPHX_RETURNS(unsafeAtomicAdd(&x, y)); - -__device__ inline void builtin_assign(half2& x, half2 y, op::sum) -{ - __builtin_amdgcn_global_atomic_fadd_v2f16(&x, y); -} - -template -constexpr bool is_aligned(const void* ptr) -{ - auto iptr = bit_cast(ptr); - return (iptr % alignof(T)) == 0; -} - -__device__ inline void builtin_assign(half& x, half y, op::sum) -{ - half* address = &x; - if(is_aligned(address)) - { - __builtin_amdgcn_global_atomic_fadd_v2f16(address, half2{half(y), half(0)}); - } - else - { - __builtin_amdgcn_global_atomic_fadd_v2f16(address - 1, half2{half(0), half(y)}); - } -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto builtin_assign(T& x, T y, op::min) - MIGRAPHX_RETURNS(unsafeAtomicMin(&x, y)); - -template -MIGRAPHX_DEVICE_CONSTEXPR auto builtin_assign(T& x, T y, op::max) - MIGRAPHX_RETURNS(unsafeAtomicMax(&x, y)); - -template -MIGRAPHX_DEVICE_CONSTEXPR auto builtin_assign(vec& x, vec y, Op op) - -> decltype(builtin_assign(x[0], y[0], op), void()) -{ - for(index_int i = 0; i < N; i++) - { - builtin_assign(x[i], y[i], op); - } -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto assign(rank<0>, T& x, T y, Op op) - MIGRAPHX_RETURNS(cas(cas_rank{}, x, y, op)); - -template -MIGRAPHX_DEVICE_CONSTEXPR auto assign(rank<1>, T& x, T y, Op op) - MIGRAPHX_RETURNS(builtin_assign(x, y, op)); - -} // namespace atomic - -template -MIGRAPHX_DEVICE_CONSTEXPR void atomic_assign(T& x, U y, Op op) -{ - atomic::assign(rank<1>{}, x, T(y), op); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_ATOMIC_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/bit_cast.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/bit_cast.hpp deleted file mode 100644 index e559658a0..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/bit_cast.hpp +++ /dev/null @@ -1,42 +0,0 @@ -/* ************************************************************************ - * Copyright (C) 2016-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- - * ies of the Software, and to permit persons to whom the Software is furnished - * to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- - * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- - * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ************************************************************************ */ -#ifndef MIGRAPHX_GUARD_KERNELS_BITCAST_HPP -#define MIGRAPHX_GUARD_KERNELS_BITCAST_HPP - -#include -#include - -namespace migraphx { - -template {} and is_trivially_copyable{})> -inline constexpr auto bit_cast(From fr) noexcept -{ - return vec_transform(fr)([](auto x) -> To { - static_assert(sizeof(To) == sizeof(decltype(x))); - return __builtin_bit_cast(To, x); - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_BITCAST_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck.hpp deleted file mode 100644 index de22e7b07..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck.hpp +++ /dev/null @@ -1,175 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_CK_HPP -#define MIGRAPHX_GUARD_KERNELS_CK_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -namespace detail { -template -struct to_ck_type_impl -{ - using type = T; -}; -template <> -struct to_ck_type_impl -{ - using type = ck::half_t; -}; - -template -struct to_ck_type_impl -{ - using type = const typename to_ck_type_impl::type; -}; - -template -constexpr bool is_row_major() -{ - constexpr auto strides = Shape{}.strides; - MIGRAPHX_ASSERT(strides.size() >= 2); - if(strides.back() == 1) - { - MIGRAPHX_ASSERT(not Shape{}.is_transposed()); - return true; - } - MIGRAPHX_ASSERT(strides[strides.size() - 2] == 1); - - return false; -} - -} // namespace detail - -template -using to_ck_type = typename detail::to_ck_type_impl::type; - -template -constexpr auto to_ck_pointer(T* x) -{ - return static_cast*>(x); -} - -template -constexpr auto to_ck_const_pointer(const T* x) -{ - return static_cast*>(x); -} - -template -using to_ck_gemm_layout = conditional_t>(), - ck::tensor_layout::gemm::RowMajor, - ck::tensor_layout::gemm::ColumnMajor>; - -template -constexpr auto to_ck_tensor() -{ - constexpr auto s = get_shape_c{}; - return sequence(s.lens.size(), [&](auto... is) { - return ck::make_naive_tensor_descriptor(ck::make_tuple(s.lens[is]...), - ck::make_tuple(s.strides[is]...)); - }); -} - -template -struct ck_function_adaptor : F -{ - template - constexpr ck_function_adaptor(Ts&&... xs) : F(static_cast(xs)...) - { - } - - template - constexpr void operator()(T& out, Ts&&... xs) const - { - out = static_cast(*this)(static_cast(xs)...); - } -}; - -struct ck_nop -{ - template - constexpr void operator()(T&) const - { - } -}; - -struct ck_passthrough -{ - template - constexpr void operator()(T& y, U x) const - { - y = x; - } -}; - -struct ck_scale -{ - constexpr ck_scale(float s) : scale(s) {} - - template - constexpr void operator()(T& y, U x) const - { - y = x * static_cast(scale); - } - - float scale; -}; - -struct ck_add -{ - template - constexpr void operator()(T& y, U x) const - { - y += x; - } -}; - -// In CK, the B matrix is ordered as N,K instead of K,N -template -constexpr auto ck_transposeb_dims(Dims dims) -{ - return unpack(dims, [](auto k, auto n) { return make_const_array(n, k); }); -} - -template -using ck_transposeb = decltype(make_shape(ck_transposeb_dims(get_shape_c{}.lens), - ck_transposeb_dims(get_shape_c{}.strides))); - -#ifdef MIGRAPHX_CK_CHECK -#define MIGRAPHX_CK_STATIC_ASSERT static_assert -#else -#define MIGRAPHX_CK_STATIC_ASSERT(...) -#endif - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_CK_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck_gemm.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck_gemm.hpp deleted file mode 100644 index ccef52132..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck_gemm.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_CK_GEMM_HPP -#define MIGRAPHX_GUARD_KERNELS_CK_GEMM_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -template -__device__ void ck_gemm_matrix(E e, A a, B b, Ds... ds) -{ - constexpr auto desc = G::make_descriptor(to_ck_tensor(), - to_ck_tensor>(), - ck::make_tuple(to_ck_tensor()...), - to_ck_tensor()); - - MIGRAPHX_STATIC_ASSERT_FOR(desc.IsValid()) - { - G::Run(desc, - to_ck_const_pointer(a.data()), - to_ck_const_pointer(b.data()), - ck::make_tuple(to_ck_const_pointer(ds.data())...), - to_ck_pointer(e.data())); - } -} - -template -__device__ void ck_gemm(Ts... xs) -{ - gemm_batch_args(make_index(), _c, xs...)( - [](auto... ys) { ck_gemm_matrix(ys...); }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck_gemm_softmax_gemm.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck_gemm_softmax_gemm.hpp deleted file mode 100644 index 8e381f375..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ck_gemm_softmax_gemm.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_CK_GEMM_SOFTMAX_GEMM_HPP -#define MIGRAPHX_GUARD_KERNELS_CK_GEMM_SOFTMAX_GEMM_HPP - -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -template -struct ck_gemm_softmax_gemm_settings -{ - T scale{}; -}; - -template -constexpr ck_gemm_softmax_gemm_settings make_ck_gemm_softmax_gemm_settings(Ts... xs) -{ - return {xs...}; -} - -template -__device__ void ck_gemm_softmax_gemm_matrix(C c, A a, B b, B1 b1, Settings s) -{ - constexpr auto desc = G::make_descriptor(to_ck_tensor(), - to_ck_tensor>(), - to_ck_tensor>(), - to_ck_tensor()); - - MIGRAPHX_STATIC_ASSERT_FOR(desc.IsValid()) - { - G::Run(desc, - s.scale, - to_ck_const_pointer(a.data()), - to_ck_const_pointer(b.data()), - to_ck_const_pointer(b1.data()), - to_ck_pointer(c.data())); - } -} - -template -__device__ void ck_gemm_softmax_gemm(Settings s, Ts... xs) -{ - gemm_batch_args(make_index(), _c, xs...)( - [&](auto... ys) { ck_gemm_softmax_gemm_matrix(ys..., s); }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/compute_attention_probabilities.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/compute_attention_probabilities.hpp deleted file mode 100644 index 5838d9874..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/compute_attention_probabilities.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_COMPUTE_ATTENTION_PROBABILITIES_HPP -#define MIGRAPHX_GUARD_KERNELS_COMPUTE_ATTENTION_PROBABILITIES_HPP - -#include -#include -#include - -namespace migraphx { - -template -__device__ void -calculate_attention_probs(AttnProbs attention_probs, // output buffer with size BxNxSxT - Query query, // Q data. Its size is BxNxSxH - SeqLensK seqlens_k, // past sequence lengths tensor - PresentKey present_key, // present key only - Params params, - index_int idx) -{ - const index_int batch_size = params.batch_size; - const index_int sequence_length = params.sequence_length; - const index_int head_size = params.head_size; - const index_int present_buffer_sequence_length = params.seqlen_present_kv_cache; - const index_int num_heads = params.num_heads; - const index_int kv_num_heads = params.kv_num_heads; - const index_int packed_batch_stride = - (num_heads + 2 * kv_num_heads) * sequence_length * head_size; - const index_int kv_num_heads_factor = num_heads / kv_num_heads; - const index_int q_input_chunk_length = sequence_length * head_size; // S x H - const index_int present_buff_chunk_length = present_buffer_sequence_length * head_size; // T x H - - const index_int loop_len = batch_size * num_heads; - const float alpha = - params.scale == 0.0f ? 1.0f / sqrt(static_cast(head_size)) : params.scale; - - const index_int i = idx / (sequence_length * present_buffer_sequence_length); - const index_int inner_i = idx % (sequence_length * present_buffer_sequence_length); - if(i < loop_len) - { - const auto batch_index = i / num_heads; - const auto head_index = i % num_heads; - const index_int total_seqlen = seqlens_k[batch_index] + 1; - const index_int output_offset = i * sequence_length * present_buffer_sequence_length; - auto output = attention_probs + output_offset; - auto pk = present_key + ((i / kv_num_heads_factor) * present_buff_chunk_length); - auto q = query + packed_batch_stride * batch_index + q_input_chunk_length * head_index; - - naive_gemm gemm{sequence_length, - total_seqlen, - head_size, - head_size, - head_size, - present_buffer_sequence_length, - true, - alpha, - 0.0f}; - gemm.compute(output, q, pk, inner_i); - } -} - -template -__device__ void compute_attention_probabilities(Output output, - Query query, - PresentKey present_key, - PresentValue, - SeqLensK seqlens_k, - Params params) -{ - auto ind = make_index(); - ind.global_stride( - params.batch_size * params.num_heads * params.sequence_length * - params.seqlen_present_kv_cache, - [&](auto idx) { - calculate_attention_probs( - output.begin(), query.begin(), seqlens_k.begin(), present_key.begin(), params, idx); - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/compute_attention_scores.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/compute_attention_scores.hpp deleted file mode 100644 index f51cfacd5..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/compute_attention_scores.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_COMPUTE_ATTENTION_SCORES_HPP -#define MIGRAPHX_GUARD_KERNELS_COMPUTE_ATTENTION_SCORES_HPP - -#include -#include -#include - -namespace migraphx { - -template -__device__ void -calculate_attention_score(Output output, // buffer for the result with size BxSxNxH - const AttnProbs attention_probs, // Attention probs with size BxNxSxT - const SeqLensK seqlens_k, // past sequence lengths tensor - PresentValue present_value, // present value only - Params params, - index_int idx) -{ - const index_int batch_size = params.batch_size; - const index_int num_heads = params.num_heads; - const index_int sequence_length = params.sequence_length; - const index_int head_size = params.head_size; - const index_int hidden_size = params.hidden_size; - const index_int present_buffer_sequence_length = params.seqlen_present_kv_cache; - const index_int kv_num_heads = params.kv_num_heads; - const index_int kv_num_heads_factor = num_heads / kv_num_heads; - const index_int present_buff_chunk_length = present_buffer_sequence_length * head_size; // T x H - - auto loop_len = batch_size * num_heads; - const index_int i = idx / (sequence_length * head_size); - const index_int inner_i = idx % (sequence_length * head_size); - if(i < loop_len) - { - const index_int batch_index = i / num_heads; - const index_int head_index = i % num_heads; - const index_int total_seqlen = seqlens_k[batch_index] + 1; - - auto pv = present_value + ((i / kv_num_heads_factor) * present_buff_chunk_length); - Output output_current = - output + (batch_index * sequence_length * num_heads + head_index) * head_size; - ptrdiff_t attention_probs_offset = sequence_length * present_buffer_sequence_length * i; - - naive_gemm gemm{sequence_length, - head_size, - total_seqlen, - present_buffer_sequence_length, - head_size, - hidden_size, - false, - 1.0f, - 0.0f}; - gemm.compute(output_current, attention_probs + attention_probs_offset, pv, inner_i); - } -} - -template -__device__ void compute_attention_scores(Output output, - Query, - PresentKey, - PresentValue present_value, - SeqLensK seqlens_k, - AttnProbs attn_probs, - Params params) -{ - const index_int elements = - params.batch_size * params.num_heads * params.sequence_length * params.head_size; - auto ind = make_index(); - ind.global_stride(elements, [&](auto idx) { - calculate_attention_score(output.begin(), - attn_probs.begin(), - seqlens_k.begin(), - present_value.begin(), - params, - idx); - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/concat.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/concat.hpp deleted file mode 100644 index 9dd2ec6b6..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/concat.hpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include - -#ifndef MIGRAPHX_GUARD_KERNELS_CONCAT_HPP -#define MIGRAPHX_GUARD_KERNELS_CONCAT_HPP - -namespace migraphx { - -template -constexpr auto concat_slice(Output out, Input, Start) -{ - constexpr auto lens = get_shape_c{}.lens; - constexpr auto strides = get_shape_c{}.strides; - constexpr auto offset = return_c([] { - constexpr auto output_shape = get_shape_c{}; - return Start{} * output_shape.strides[Axis]; - }); - constexpr auto s = make_shape(lens, strides); - MIGRAPHX_ASSERT(offset < out.get_shape().element_space()); - MIGRAPHX_ASSERT((s.element_space() + offset) <= out.get_shape().element_space()); - return make_tensor_view(out.data() + offset, s); -} - -template -constexpr auto concat_slices(Input input, Start start, Ts... xs) -{ - return [=](auto f) { return f(concat_slice(xs, input, start)...); }; -} - -template -constexpr auto concat_ends(Input) -{ - constexpr auto lens = get_shape_c{}.lens; - return _c; -} - -template -__device__ auto concat_each(index idx, Start start, InputPack input_pack, F f, Ts... ts) -{ - return input_pack([&](auto g, auto x, auto... xs) { - return concat_slices(x, start, ts...)([&](auto z, auto... ys) { - idx.global_stride(x.get_shape().elements(), - [&](auto i) { z[i] = f(g(x[i], xs[i]...), ys[i]...); }); - - return start + concat_ends(x); - }); - }); -} - -template -__device__ auto concat(InputPacks... input_packs) -{ - return [=](auto f, auto... ts) { - auto idx = make_index(); - fold([&](auto start, auto input_pack) { - return concat_each(idx, start, input_pack, f, ts...); - })(_c<0>, input_packs...); - }; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_CONCAT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/concat_past_present.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/concat_past_present.hpp deleted file mode 100644 index dcfebbbe0..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/concat_past_present.hpp +++ /dev/null @@ -1,141 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_CONCAT_PAST_PRESENT_HPP -#define MIGRAPHX_GUARD_KERNELS_CONCAT_PAST_PRESENT_HPP - -#include -#include -#include - -namespace migraphx { - -template -__device__ void copy_data(Dest destination, const Src source, index_int n, index_int idx) -{ - if(idx < n) - { - destination[idx] = source[idx]; - } -} - -struct concat_state_chunk -{ - index_int present_buff_chunk_length; - index_int past_buff_chunk_length; - index_int past_chunk_length; - index_int new_chunk_length; - bool is_prompt; - bool past_present_share_buffer; - std::ptrdiff_t i; - - template - __device__ Present compute(Past past, const Chunk chunk, Present present, index_int idx) - { - auto start = present + i * present_buff_chunk_length; - - auto p = start; - if(not is_prompt) - { - if(not past_present_share_buffer) - { - const auto src_past = past + i * past_buff_chunk_length; - copy_data(p, src_past, past_chunk_length, idx); - } - p += past_chunk_length; - } - copy_data(p, chunk, new_chunk_length, idx); - return start; - } -}; - -template -__device__ void -update_cache(const Present present, SeqLensK seqlens_k, Cache cache, Params params, index_int idx) -{ - const index_int batch_size = params.batch_size; - const index_int sequence_length = params.sequence_length; - const index_int head_size = params.head_size; - const index_int past_buffer_sequence_length = params.seqlen_present_kv_cache; - const index_int present_buffer_sequence_length = past_buffer_sequence_length; - const index_int num_heads = params.num_heads; - const index_int kv_num_heads = params.kv_num_heads; - const bool is_prompt = sequence_length != 1; - const index_int packed_batch_stride = - (num_heads + 2 * kv_num_heads) * sequence_length * head_size; - const index_int kv_num_heads_factor = num_heads / kv_num_heads; - const index_int kv_input_chunk_length = sequence_length * head_size; // L x H - const index_int past_buff_chunk_length = past_buffer_sequence_length * head_size; // L x H - const index_int present_buff_chunk_length = present_buffer_sequence_length * head_size; // T x H - - const index_int loop_len = batch_size * num_heads; - const index_int i = idx / (sequence_length * head_size); - const index_int inner_i = idx % (sequence_length * head_size); - if(i < loop_len) - { - const index_int batch_index = i / num_heads; - const index_int head_index = i % num_heads; - const index_int past_seqlen = sequence_length == 1 - ? static_cast(seqlens_k[batch_index]) - : past_buffer_sequence_length; - const index_int past_chunk_length = past_seqlen * head_size; - - auto current = present + packed_batch_stride * batch_index + - kv_input_chunk_length * (head_index / kv_num_heads_factor); - - concat_state_chunk concat{present_buff_chunk_length, - past_buff_chunk_length, - past_chunk_length, - kv_input_chunk_length, - is_prompt, - params.past_present_share_buffer, - i / kv_num_heads_factor}; - concat.compute(cache, current, cache, inner_i); - } -} - -template -__device__ void concat_past_present( - const Query query, PastKey past_key, PastValue past_value, SeqLensK seqlens_k, Params params) -{ - auto ind = make_index(); - auto elements = - 2 * params.batch_size * params.kv_num_heads * params.sequence_length * params.head_size; - ind.global_stride(elements, [&](auto idx) { - auto q = query.begin(); - auto k = q + params.num_heads * params.sequence_length * params.head_size; - auto v = q + (params.num_heads + params.kv_num_heads) * params.sequence_length * - params.head_size; - if(idx < elements / 2) - { - update_cache(k, seqlens_k, past_key.begin(), params, idx); - } - else if(idx < elements) - { - update_cache(v, seqlens_k, past_value.begin(), params, idx - (elements / 2)); - } - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/copy.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/copy.hpp deleted file mode 100644 index 972988992..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/copy.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_COPY_HPP -#define MIGRAPHX_GUARD_KERNELS_COPY_HPP - -#include -#include - -namespace migraphx { - -template -__device__ void local_vector_copy(Index idx, T* src, U* dst, Size size) -{ - constexpr auto n = find_vectorize_size([&](auto i) { return (size % i) == 0; }); - auto vsrc = as_vec(remove_bool(src)); - auto vdst = as_vec(remove_bool(dst)); - index_int vsize = size / n; - idx.local_stride(vsize, [&](auto i) { vdst[i] = vsrc[i]; }); -} - -template -__device__ void local_tensor_copy(Index idx, T src, U dst) -{ - constexpr auto src_shape = get_shape_c{}; - constexpr auto dst_shape = get_shape_c{}; - if constexpr(src_shape == dst_shape and (src_shape.packed() or src_shape.broadcasted())) - { - local_vector_copy(idx, src.data(), dst.data(), src_shape.element_space()); - } - else - { - constexpr auto perm = find_permutation(src_shape, dst_shape); - auto new_src = reorder_tensor_view(src, perm); - auto new_dst = reorder_tensor_view(dst, perm); - auto_vectorize()(new_src, new_dst)([&](auto vsrc, auto vdst) { - index_int size = vsrc.get_shape().elements(); - idx.local_stride(size, [&](auto i) { vdst[i] = vsrc[i]; }); - }); - } -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_COPY_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/debug.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/debug.hpp deleted file mode 100644 index 5e5e16b13..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/debug.hpp +++ /dev/null @@ -1,230 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_DEBUG_HPP -#define MIGRAPHX_GUARD_KERNELS_DEBUG_HPP - -#include - -namespace migraphx { - -#define MIGRAPHX_STRINGIZE_1(...) #__VA_ARGS__ -#define MIGRAPHX_STRINGIZE(...) MIGRAPHX_STRINGIZE_1(__VA_ARGS__) - -// Workaround hip's broken abort on device code -#ifdef __HIP_DEVICE_COMPILE__ -// NOLINTNEXTLINE -#define MIGRAPHX_HIP_NORETURN -#else -// NOLINTNEXTLINE -#define MIGRAPHX_HIP_NORETURN [[noreturn]] -#endif - -namespace debug { -struct swallow -{ - template - constexpr swallow(Ts&&...) - { - } -}; - -template -struct print_buffer -{ - char buffer[N + 1] = {0}; - char* pos = buffer; - - constexpr void append(char c) - { - if(c == 0) - return; - if(pos < buffer + N) - { - *pos = c; - pos++; - } - } - static constexpr void reverse(char* first, char* last) - { - if(first == last) - return; - last--; - while(first < last) - { - char tmp = *first; - *first = *last; - *last = tmp; - first++; - last--; - } - } - - template - constexpr void append(T i) - { - if(i < 0) - { - append('-'); - i = -i; - } - if(i == 0) - { - append('0'); - return; - } - char* start = pos; - while(i != 0) - { - char c = (i % 10) + '0'; - append(c); - i = i / 10; - } - reverse(start, pos); - } - - constexpr void append(const char* str) - { - if(str == nullptr) - return; - int i = 512; - while(*str != 0 and i > 0) - { - append(*str); - str++; - i--; - } - } - - template - constexpr void append(const char (&array)[M]) - { - for(int i = 0; i < M; i++) - append(array[i]); - } -}; - -template -__host__ __device__ void print(const Ts&... xs) -{ - print_buffer<1024> buffer; - swallow{(buffer.append(xs), 0)...}; - printf("%s", buffer.buffer); -} - -} // namespace debug - -struct source_location -{ - int line = __builtin_LINE(); - const char* file = __builtin_FILE(); - const char* function = __builtin_FUNCTION(); -}; - -template -struct source_location_capture -{ - T x; - source_location loc; - // declval is a workaround since default constructor for "U" is not working with rocm-5.6 - template - static U&& declval(); - template ()))> - constexpr source_location_capture(U px, source_location ploc = source_location{}) - : x(px), loc(ploc) - { - } - template ()))> - constexpr source_location_capture(source_location_capture slc) : x(slc.x), loc(slc.loc) - { - } - - constexpr operator source_location() const { return loc; } - - constexpr operator T() const { return x; } -}; - -template -constexpr auto capture_transform(source_location_capture slc, F f) -{ - auto r = f(slc.x); - return source_location_capture(r, slc.loc); -} - -template -constexpr auto capture_transform(T x, F f) -{ - return f(x); -} - -// noreturn cannot be used on this function because abort in hip is broken -template -MIGRAPHX_HIP_NORETURN inline __host__ __device__ void -assert_fail(const T1& assertion, const T2& file, const T3& line, const T4& function) -{ - // printf is broken on hip with more than one argument, so use a simple print functions instead - debug::print(file, ":", line, ": ", function, ": assertion '", assertion, "' failed.\n"); - // printf("%s:%s: %s: assertion '%s' failed.\n", file, line, function, assertion); - abort(); -} - -template -MIGRAPHX_HIP_NORETURN inline __host__ __device__ void assert_fail(const source_location& loc, - Ts... xs) -{ - debug::print(loc.file, ":", loc.line, ": ", loc.function, ": error: ", xs..., "\n"); - abort(); -} - -// NOLINTNEXTLINE -#define MIGRAPHX_ASSERT_FAIL(cond, ...) \ - ((cond) ? void(0) : [](auto&&... private_migraphx_xs) { \ - assert_fail(private_migraphx_xs...); \ - }(__VA_ARGS__)) - -// NOLINTNEXTLINE -#define MIGRAPHX_CHECK(cond) \ - MIGRAPHX_ASSERT_FAIL(cond, #cond, __FILE__, __LINE__, __PRETTY_FUNCTION__) - -#ifdef MIGRAPHX_DEBUG -// NOLINTNEXTLINE -#define MIGRAPHX_CAPTURE_SOURCE_LOCATION(T) source_location_capture -#define MIGRAPHX_WARN(cond, loc, ...) MIGRAPHX_ASSERT_FAIL(cond, loc, __VA_ARGS__) -#define MIGRAPHX_ASSERT MIGRAPHX_CHECK -#define MIGRAPHX_ASSUME MIGRAPHX_CHECK -#define MIGRAPHX_UNREACHABLE() MIGRAPHX_ASSERT(false) -#else -// NOLINTNEXTLINE -#define MIGRAPHX_CAPTURE_SOURCE_LOCATION(T) T -#define MIGRAPHX_ASSUME __builtin_assume -#define MIGRAPHX_UNREACHABLE __builtin_unreachable -#define MIGRAPHX_ASSERT(cond) -#define MIGRAPHX_WARN(...) -#endif - -#define MIGRAPHX_STATIC_ASSERT_FOR(...) \ - static_assert(__VA_ARGS__); \ - if constexpr(__VA_ARGS__) - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_DEBUG_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/dfor.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/dfor.hpp deleted file mode 100644 index d8255d4b9..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/dfor.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_DFOR_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_DFOR_HPP - -namespace migraphx { - -// Multidimensional for loop -inline constexpr auto dfor() -{ - return [](auto f) { f(); }; -} - -template -constexpr auto dfor(T x, Ts... xs) -{ - return [=](auto f) { - for(T i = 0; i < x; i++) - { - dfor(xs...)([&](Ts... is) { f(i, is...); }); - } - }; -} - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/dpp.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/dpp.hpp deleted file mode 100644 index 5ae4c6866..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/dpp.hpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_DPP_HPP -#define MIGRAPHX_GUARD_KERNELS_DPP_HPP - -#include -#include -#include - -namespace migraphx { - -constexpr bool is_power_of_2(unsigned int x) { return x > 0 and (x & (x - 1)) == 0u; } - -#ifndef MIGRAPHX_HAS_DPP -#define MIGRAPHX_HAS_DPP 1 -#endif - -#if MIGRAPHX_HAS_DPP -constexpr unsigned int dpp_row_shr(unsigned int x) { return 0x110u | x; } - -constexpr unsigned int dpp_row_bcast(unsigned int x) -{ - unsigned int y = 0; - switch(x) - { - case 15: y = 0x142; break; - case 31: y = 0x143; break; - default: MIGRAPHX_UNREACHABLE(); - } - return y; -} - -template -__device__ T dpp_op(T& x, F f) -{ - static const index_int n = sizeof(T) < 4 ? 1 : sizeof(T) / 4; - union type - { - uint32_t reg[n]; - T data; - }; - type output{}; - type input{}; - // cppcheck-suppress unreadVariable - input.data = x; - for(index_int i = 0; i < n; i++) - { - output.reg[i] = f(input.reg[i]); - } - return output.data; -} - -template -__device__ T dpp_mov(T& x) -{ - return dpp_op(x, - [](auto i) { return __hip_move_dpp(i, DppCtrl, RowMask, BankMask, BoundCtrl); }); -} - -template -__device__ T dpp_swizzle(T& x) -{ - return dpp_op(x, [](auto i) { return __hip_ds_swizzle(i, Mask); }); -} - -template -__device__ T readlane(T& x) -{ - static_assert(is_power_of_2(Width), "Width must be a power of 2"); - return dpp_op(x, [](auto i) { return __shfl(i, SrcLane, Width); }); -} - -#endif // MIGRAPHX_HAS_DPP - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_DPP_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/float8.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/float8.hpp deleted file mode 100644 index 8227ae220..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/float8.hpp +++ /dev/null @@ -1,567 +0,0 @@ -/* ************************************************************************ - * - * The MIT License (MIT) - * - * Copyright (C) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- - * ies of the Software, and to permit persons to whom the Software is furnished - * to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- - * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- - * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ************************************************************************ */ - -#ifndef MIGRAPHX_GUARD_KERNELS_FLOAT8_HPP -#define MIGRAPHX_GUARD_KERNELS_FLOAT8_HPP -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wfloat-equal" -#pragma clang diagnostic ignored "-Wc++20-extensions" // required for "asm" inside constexpr -#endif // __clang__ - -// We are clipping in down conversion by default -#define MIGRAPHX_F8_DOWNCAST_CLIPPING 1 // NOLINT - -#include -#include -#include - -namespace migraphx { -namespace fp8 { - -enum class rounding_mode -{ - standard, // standard rounding is doing RNE -- round to nearest even - stochastic -}; - -enum class f8_type -{ - bf8 = 0, // s1e5m2 - fp8 = 1 // s1e4m3 -}; - -template -class numeric_limits; - -template -struct float8 -{ - uint8_t data; - // default constructor - __device__ constexpr float8() = default; - // default copy constructor - __device__ constexpr float8(const float8& y) = default; - struct from_bits_t - { - }; - static constexpr __device__ from_bits_t from_bits() { return from_bits_t(); } - - __device__ explicit constexpr float8(uint8_t bits, from_bits_t) : data(bits) {} - -#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) - // device specific optimized F8 down-conversion code - - template - static __device__ uint8_t cast_to_f8fnuz_from_f32(float v, uint32_t rng = 0) - { - uint8_t i8data = 0x00; - union - { - float fval; - uint32_t i32val; - uint8_t i8val[4]; // NOTE: not endian independent - } val; - - uint32_t ival = 0; - val.fval = v; - -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - if((val.i32val & 0x7F800000) != 0x7F800000) /// propagate NAN/INF, no clipping - val.fval = __builtin_amdgcn_fmed3f(val.fval, 240.0, -240.0); - } - else - { - if((val.i32val & 0x7F800000) != 0x7F800000) // propagate NAN/INF, no clipping - val.fval = __builtin_amdgcn_fmed3f(val.fval, 57344.0, -57344.0); - } -#endif - if(stochastic_rounding) - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - ival = __builtin_amdgcn_cvt_sr_fp8_f32(val.fval, rng, ival, 0); // 0 pos - } - else - { - ival = __builtin_amdgcn_cvt_sr_bf8_f32(val.fval, rng, ival, 0); // 0 pos - } - } - else // RNE CVT - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - ival = __builtin_amdgcn_cvt_pk_fp8_f32( - val.fval, val.fval, ival, false); // false -> WORD0 - } - else - { - ival = __builtin_amdgcn_cvt_pk_bf8_f32( - val.fval, val.fval, ival, false); // false -> WORD0} - } - } - val.i32val = ival; - i8data = val.i8val[0]; // little endian - - return i8data; - } -#endif // __gfx940__ - - // constructor from float -#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) - - // NOTE: ON-DEVICE... always optimal bias - explicit constexpr __device__ - float8(const float v, - migraphx::fp8::rounding_mode rm = migraphx::fp8::rounding_mode::standard, - uint32_t rng = 0) - { - if(__builtin_is_constant_evaluated() or !FNUZ) - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<3, 4, float, FNUZ /*negative_zero_nan*/, true /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#else // MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<3, 4, float, FNUZ /*negative_zero_nan*/, false /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#endif // MIGRAPHX_F8_DOWNCAST_CLIPPING - } - else - { -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<2, 5, float, FNUZ /*negative_zero_nan*/, true /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#else // MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<2, 5, float, FNUZ /*negative_zero_nan*/, false /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#endif // MIGRAPHX_FP8_DOWNCAST_CLIPPING} - } - } - else - { - // runtime branch, use cast_to_f8fnuz_from_f32 if want to avoid it - if(rm == migraphx::fp8::rounding_mode::stochastic) - data = cast_to_f8fnuz_from_f32(v, rng); - else - data = cast_to_f8fnuz_from_f32(v); - } - } -#else - // DEVICE for non-gfx940 using s/w simulation - explicit constexpr __device__ - float8(const float v, - migraphx::fp8::rounding_mode rm = migraphx::fp8::rounding_mode::standard, - uint32_t rng = 0) - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<3, 4, float, FNUZ /*negative_zero_nan*/, true /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#else // MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<3, 4, float, FNUZ /*negative_zero_nan*/, false /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#endif // MIGRAPHX_F8_DOWNCAST_CLIPPING - } - else - { -#ifdef MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<2, 5, float, FNUZ /*negative_zero_nan*/, true /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#else // MIGRAPHX_F8_DOWNCAST_CLIPPING - data = migraphx::fp8::impl:: - cast_to_f8<2, 5, float, FNUZ /*negative_zero_nan*/, false /*clip*/>( - v, (rm == migraphx::fp8::rounding_mode::stochastic), rng); -#endif // MIGRAPHX_FP8_DOWNCAST_CLIPPING} - } - } -#endif // __gfx940___ - - // Constructor from half - explicit constexpr __device__ - float8(const _Float16 v, rounding_mode rm = rounding_mode::standard, uint32_t rng = 0) - : float8(static_cast(v), rm, rng) - { - } - - // constructor from int - explicit constexpr __device__ - float8(const int v, rounding_mode rm = rounding_mode::standard, uint32_t rng = 0) - : float8(static_cast(v), rm, rng) - { - } - - // constructor from uint - explicit constexpr __device__ - float8(const uint32_t v, rounding_mode rm = rounding_mode::standard, uint32_t rng = 0) - : float8(static_cast(v), rm, rng) - { - } - - // constructor from double - explicit constexpr __device__ - float8(const double v, rounding_mode rm = rounding_mode::standard, uint32_t rng = 0) - : float8(static_cast(v), rm, rng) - { - } - - // constructor from bool - explicit constexpr __device__ - float8(const bool v, rounding_mode rm = rounding_mode::standard, uint32_t rng = 0) - : float8(static_cast(v), rm, rng) - { - } - // convert to float -#if defined(__gfx940__) || defined(__gfx941__) || defined(__gfx942__) // NOLINT - // upcast using device specific intrinsic - inline constexpr __device__ operator float() const - { - if(__builtin_is_constant_evaluated() or !FNUZ) - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - return migraphx::fp8::impl::cast_from_f8<3, 4, float, FNUZ /*negative_zero_nan*/>( - data); - } // else - return migraphx::fp8::impl::cast_from_f8<2, 5, float, FNUZ /*negative_zero_nan*/>(data); - } - else - { - float fval = 0; - uint32_t i32val = static_cast(data); - - // upcast - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - __asm__ volatile("v_cvt_f32_fp8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val)); - } - else - { - __asm__ volatile("v_cvt_f32_bf8 %0, %1 src0_sel:BYTE_0" : "=v"(fval) : "v"(i32val)); - } - - return fval; - } - } - -#else // non gfx940 - inline constexpr __device__ operator float() const - { - if constexpr(T == migraphx::fp8::f8_type::fp8) - { - return migraphx::fp8::impl::cast_from_f8<3, 4, float, FNUZ /*negative_zero_nan*/>(data); - } // else - return migraphx::fp8::impl::cast_from_f8<2, 5, float, FNUZ /*negative_zero_nan*/>(data); - } -#endif - - inline constexpr explicit __device__ operator bool() const { return not is_zero(); } - - // check for zero - inline __device__ constexpr bool is_zero() const - { - if constexpr(FNUZ) - { - return data == 0x00; - } - else - { - return (data == 0x00) or (data == 0x80); - } - } - - // check for nan - inline __device__ constexpr bool is_nan() const - { - if constexpr(FNUZ) - { - return data == 0x80; - } - else - { - if(T == migraphx::fp8::f8_type::bf8) - { - return (data == 0x7D) or (data == 0x7E) or (data == 0x7F) or (data == 0xFD) or - (data == 0xFE) or (data == 0xFF); - } - else - { - return (data == 0x7F) or (data == 0xFF); - } - } - } - - // check for inf - inline __device__ constexpr bool is_inf() const - { - if constexpr(FNUZ) - { - return data == 0x80; - } - else - { - if(T == migraphx::fp8::f8_type::bf8) - { - return (data == 0x7C) or (data == 0xFC); - } - else - { - // no infinities in e4m3fn, represent them as NaNs - return (data == 0x7F) or (data == 0xFF); - } - } - } - -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_SHORT_UNARY_OP(unary_op, binary_op) \ - constexpr float8& __device__ operator unary_op(const float8& rhs) \ - { \ - const auto tmp = static_cast(*this) binary_op static_cast(rhs); \ - *this = static_cast(tmp); \ - return *this; \ - } \ - constexpr float8& __device__ operator unary_op(const float& rhs) \ - { \ - const auto tmp = static_cast(*this) binary_op static_cast(rhs); \ - *this = static_cast(tmp); \ - return *this; \ - } - - MIGRAPHX_FP8_SHORT_UNARY_OP(*=, *) - MIGRAPHX_FP8_SHORT_UNARY_OP(-=, -) - MIGRAPHX_FP8_SHORT_UNARY_OP(+=, +) - MIGRAPHX_FP8_SHORT_UNARY_OP(/=, /) - - inline __device__ constexpr float8& operator=(const float8& rhs) = default; - inline __device__ constexpr float8& operator=(float8&& rhs) noexcept = default; - - inline __device__ constexpr bool operator<(const float8& rhs) const - { - const auto we = static_cast(*this); - const auto them = static_cast(rhs); - return we < them; - } - - inline __device__ constexpr bool operator>(const float8& rhs) const - { - const auto we = static_cast(*this); - const auto them = static_cast(rhs); - return we > them; - } -}; - -// https://onnx.ai/onnx/technical/float8.html -using fp8e4m3fn = float8; -using fp8e5m2 = float8; -using fp8e4m3fnuz = float8; -using fp8e5m2fnuz = float8; - -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_BINARY_OP(binary_op, T, U) \ - inline constexpr U __device__ operator binary_op(const T& lhs, const T& rhs) \ - { \ - return U(static_cast(lhs) binary_op static_cast(rhs)); \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_OTHER_OPS(T) \ - inline constexpr __device__ T fabs(T v) \ - { \ - /*NOLINTNEXTLINE*/ \ - v.data = v.data & 0x7f; \ - return v; \ - } \ - inline __device__ constexpr bool operator==(const T& lhs, const T& rhs) \ - { \ - if(rhs.is_nan() or rhs.is_inf() or lhs.is_nan() or lhs.is_inf()) \ - return false; \ - else if((rhs.is_zero() and lhs.is_zero()) or (lhs.data == rhs.data)) \ - return true; \ - return false; \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_FP8_GEN_OP_OVERLOADS(T) \ - MIGRAPHX_FP8_BINARY_OP(*, T, T) \ - MIGRAPHX_FP8_BINARY_OP(-, T, T) \ - MIGRAPHX_FP8_BINARY_OP(/, T, T) \ - MIGRAPHX_FP8_BINARY_OP(+, T, T) \ - MIGRAPHX_FP8_BINARY_OP(>=, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(<=, T, bool) \ - MIGRAPHX_FP8_BINARY_OP(!=, T, bool) \ - MIGRAPHX_FP8_OTHER_OPS(T) - -MIGRAPHX_FP8_GEN_OP_OVERLOADS(fp8e5m2) -MIGRAPHX_FP8_GEN_OP_OVERLOADS(fp8e5m2fnuz) -MIGRAPHX_FP8_GEN_OP_OVERLOADS(fp8e4m3fn) -MIGRAPHX_FP8_GEN_OP_OVERLOADS(fp8e4m3fnuz) - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = false; - static constexpr __device__ fp8e4m3fnuz epsilon() - { - return fp8e4m3fnuz(0x28, fp8e4m3fnuz::from_bits()); - } - // NOLINTNEXTLINE - static constexpr __device__ fp8e4m3fnuz quiet_NaN() - { - return fp8e4m3fnuz(0x80, fp8e4m3fnuz::from_bits()); - } - - static constexpr __device__ fp8e4m3fnuz max() - { - return fp8e4m3fnuz(0x7F, fp8e4m3fnuz::from_bits()); - } - // this is min value that is not DeNormalized(DeNorm). DeNorm min is 0x01 - static constexpr __device__ fp8e4m3fnuz min() - { - return fp8e4m3fnuz(0x08, fp8e4m3fnuz::from_bits()); - } - - static constexpr __device__ fp8e4m3fnuz lowest() - { - return fp8e4m3fnuz(0xFF, fp8e4m3fnuz::from_bits()); - } -}; - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = false; - static constexpr __device__ fp8e4m3fn epsilon() - { - return fp8e4m3fn(0x20, fp8e4m3fn::from_bits()); - } - // NOLINTNEXTLINE - static constexpr __device__ fp8e4m3fn quiet_NaN() - { - return fp8e4m3fn(0x7F, fp8e4m3fn::from_bits()); - } - - static constexpr __device__ fp8e4m3fn max() { return fp8e4m3fn(0x7E, fp8e4m3fn::from_bits()); } - // this is min value that is not DeNormalized(DeNorm). DeNorm min is 0x01 - static constexpr __device__ fp8e4m3fn min() { return fp8e4m3fn(0x08, fp8e4m3fn::from_bits()); } - - static constexpr __device__ fp8e4m3fn lowest() - { - return fp8e4m3fn(0xFE, fp8e4m3fn::from_bits()); - } -}; - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = false; - static constexpr __device__ fp8e5m2fnuz epsilon() - { - return fp8e5m2fnuz(0x34, fp8e5m2fnuz::from_bits()); - } - - static constexpr __device__ fp8e5m2fnuz quiet_NaN() // NOLINT - { - return fp8e5m2fnuz(0x80, fp8e5m2fnuz::from_bits()); - } - - static constexpr __device__ fp8e5m2fnuz max() - { - return fp8e5m2fnuz(0x7F, fp8e5m2fnuz::from_bits()); - } - // this is min value that is not DeNormalized(DeNorm). DeNorm min is 0x01. - static constexpr __device__ fp8e5m2fnuz min() - { - return fp8e5m2fnuz(0x4, fp8e5m2fnuz::from_bits()); - } - - static constexpr __device__ fp8e5m2fnuz lowest() - { - return fp8e5m2fnuz(0xFF, fp8e5m2fnuz::from_bits()); - } -}; - -template <> -class numeric_limits -{ - public: - static constexpr bool has_infinity = true; - static constexpr __device__ fp8e5m2 epsilon() { return fp8e5m2(0x34, fp8e5m2::from_bits()); } - // 7D, 7E, 7F are positive NaNs and FD, FE, FF are negative NaNs - static constexpr __device__ fp8e5m2 quiet_NaN() // NOLINT - { - return fp8e5m2(0xFF, fp8e5m2::from_bits()); - } - - static constexpr __device__ fp8e5m2 max() { return fp8e5m2(0x7B, fp8e5m2::from_bits()); } - // this is min value that is not DeNormalized(DeNorm). DeNorm min is 0x01. - static constexpr __device__ fp8e5m2 min() { return fp8e5m2(0x4, fp8e5m2::from_bits()); } - - static constexpr __device__ fp8e5m2 lowest() { return fp8e5m2(0xFB, fp8e5m2::from_bits()); } - // 7C and FC both are infinity - static constexpr __device__ fp8e5m2 infinity() { return fp8e5m2(0x7C, fp8e5m2::from_bits()); } -}; - -} // namespace fp8 -template {} or is_same{} or - is_same{} or is_same{})> -constexpr T numeric_max(migraphx::fp8::f8_type unused = migraphx::fp8::f8_type::fp8) -{ - // unused parameter is added to make this numeric_max different overload definition - // compared to numeric_max defined in type_traits.hpp - (void)(unused); - return fp8::numeric_limits::max(); -} -template {} or is_same{} or - is_same{} or is_same{})> -constexpr T numeric_lowest(migraphx::fp8::f8_type unused = migraphx::fp8::f8_type::fp8) -{ - // unused parameter is added to make this numeric_lowest different overload definition - // compared to numeric_lowest defined in type_traits.hpp - (void)(unused); - return fp8::numeric_limits::lowest(); -} -} // namespace migraphx -// ================================================================================================= -#if defined(__clang__) -#pragma clang diagnostic pop -#endif // __clang__ - -#endif // MIGRAPHX_GUARD_KERNELS_FLOAT8_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/float8_impl.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/float8_impl.hpp deleted file mode 100644 index 2eca5ed4a..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/float8_impl.hpp +++ /dev/null @@ -1,331 +0,0 @@ -/* ************************************************************************ - * Copyright (C) 2016-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- - * ies of the Software, and to permit persons to whom the Software is furnished - * to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- - * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- - * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - * ************************************************************************ */ - -#ifndef MIGRAPHX_GUARD_KERNELS_FP8_IMPL_HPP -#define MIGRAPHX_GUARD_KERNELS_FP8_IMPL_HPP -#include -#include -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreserved-identifier" -#endif - -namespace migraphx { - -namespace fp8 { -namespace impl { - -// NOLINTBEGIN -template -__device__ constexpr uint8_t cast_to_f8(T f_x, bool stoch = false, uint32_t rng = 0) -{ - constexpr bool is_float = true; - // half is not supported for now - constexpr bool is_half = false; - static_assert(Wm + We == 7, "Wm+We==7"); - static_assert(is_float or is_half, "Only float can be cast to f8"); - - const uint32_t mfmt = (sizeof(T) == 4) ? 23 : 10; - typename migraphx::conditional_t x; - - if constexpr(sizeof(T) == 4) - x = migraphx::bit_cast(f_x); - else - x = migraphx::bit_cast(f_x); - - uint32_t head = 0; - uint32_t mantissa = 0; - int exponent = 0; - uint32_t bias = 0; - uint32_t sign = 0; - if constexpr(sizeof(T) == 4) - { - head = x & 0xFF800000; - mantissa = x & 0x7FFFFF; - exponent = (head >> 23) & 0xFF; - sign = head >> 31; - bias = 127; - } - else - { - head = x & 0xFC00; - mantissa = x & 0x3FF; - exponent = (head >> 10) & 0x1F; - sign = head >> 15; - bias = 15; - } - - uint32_t signed_inf = (sign << 7) + (((1 << We) - 1) << Wm); - uint32_t signed_all_ones = (sign << 7) + ((((1 << We) - 1) << Wm) + ((1 << Wm) - 1)); - - // Calcualte maximum singed value FLT_MAX, FLT_MIN - uint32_t signed_max = signed_all_ones; - if(not NegativeZeroNan) - signed_max = (Wm == 2) ? (signed_max - 4) : (signed_max - 1); - - // Deal with inf and NaNs - if(NegativeZeroNan) // For the FNUZ cases, it is simple just return NaNs - { - if((sizeof(T) == 4 and ((x & 0x7F800000) == 0x7F800000)) or - (sizeof(T) == 2 and ((x & 0x7C00) == 0x7C00))) - return 0x80; - } - else - { - // calculate most common NaN mantissa for FP8, which is all Ones in binary - uint32_t nan_mantissa = 1; - for(auto i = 1; i < Wm; ++i) - { - nan_mantissa |= (nan_mantissa << 1); - } - if((sizeof(T) == 4 and ((x & 0x7F800000) == 0x7F800000)) or - (sizeof(T) == 2 and ((x & 0x7C00) == 0x7C00))) - { - // infinity - if(mantissa == 0) - { - if(sign == 0) - return (Wm == 2) ? 0x7B : 0x7E; - else - return (Wm == 2) ? 0xFB : 0xFE; - } - else // NaNs - return signed_inf + nan_mantissa; - } - } - // handle positive zero - if(x == 0) - return 0; - // handle negative zero - else if((sizeof(T) == 4 and x == 0x80000000) or (sizeof(T) == 2 and x == 0x8000)) - { - return NegativeZeroNan ? 0 : 0x80; // For FNUZ types neg zero is just positive zero - } - - /* First need to check if it is normal or denorm as there is a difference of implict 1 - Then need to adjust the exponent to align with the F8 exponent, in the meanwhile, shift - The mantissa. Then for stochastic rounding, add rng to mantissa and truncate. And for - RNE, no need to add rng. Then probably need to check whether there is carry and adjust - exponent and mantissa again*/ - - // For IEEE bias mode, the bias is 2^(k-1) -1 where k is the width of exponent bits - const int f8_bias = (1 << (We - 1u)) - 1 + (NegativeZeroNan ? 1 : 0); - const int f8_denormal_act_exponent = 1 - f8_bias; // actual exponent of f8 denormal - /* act_exponent is the actual exponent of fp32/fp16 (after subtracting bias) - f8_exponent is the converted f8 exponent with bias encoding - exponent_diff is the diff between fp32/fp16 exponent and f8 exponent, - the difference needs to be adjusted and mantissa shifted*/ - int act_exponent = 0; - int f8_exponent = 0; - int exponent_diff = 0; - - if(exponent == 0 and mantissa != 0) - { // fp32/fp16 is in denormal. - /* fp32 denormal is below 2^-127 so it is usually not a concern here, we mostly concern fp16 - here. In this case, f8 is usually in denormal. But there could be exceptions. fp16 denormal - has exponent bias 15 while bf8 with FNUZ has exponent bias 16. It means that there are some - numbers in fp16 denormal but they are bf8 (FNUZ) normals - smallest bf8 (FNUZ) normal is - 2^-15. fp16 numbers where exponent==0 (actual exponent -14) and highest bit of mantissa is 1 - are bf8 (FNUZ) normal. In this case, the fp16 mantissa should be shift left by 1 */ - act_exponent = 1 - bias; - exponent_diff = f8_denormal_act_exponent - - act_exponent; // actual exponent is exponent-bias+1 as it is denormal - } - else - { // fp32/fp16 is normal with implicit 1 - act_exponent = exponent - bias; - if(act_exponent <= f8_denormal_act_exponent) - { - /* This is the case where fp32/fp16 is normal but it is in f8 denormal range. - For example fp8 FNUZ mode, denormal exponent is -7, but if the fp32/fp16 - actual exponent is -7, it is actually larger due to the implict 1, - Therefore it needs to be adjust to -6 and mantissa shift right by 1. - So for fp32/fp16, exponent -8 is the cut point to convert to fp8 FNUZ */ - exponent_diff = f8_denormal_act_exponent - act_exponent; - } - else - { // both fp32/fp16 and f8 are in normal range - exponent_diff = - 0; // exponent_diff=0 does not mean there is no difference for this case, - // act_exponent could be larger. Just that it does not need shift mantissa - } - mantissa += (1 << mfmt); // Add the implicit 1 into mantissa - } - - // need to know whether the number is right in the middle of two adjacent fp8 numbers. use max - // value of 31 to avoid undefined behaviour - bool midpoint = (mantissa & ((1u << (mfmt - Wm + exponent_diff)) - 1)) == - (1u << (mfmt - Wm + exponent_diff - 1)); - /* This part is a bit tricky. The judgment of whether it is a tie needs to be done before we - shift right as shift right could rip off some residual part and make something not midpoint look - like midpoint. For example, the fp16 number 0x1002 (0 00100 0000000010), it is larger than - midpoint, but after shift right by 4 bits, it would look like midpoint. - */ - - if(exponent_diff > 0) - mantissa >>= exponent_diff; - else if(exponent_diff == -1) - mantissa <<= -exponent_diff; - bool implicit_one = mantissa & (1 << mfmt); - // if there is no implict 1, it means the f8 is denormal and need to adjust to denorm exponent - f8_exponent = - (act_exponent + exponent_diff) /*actual f8 exponent*/ + f8_bias - (implicit_one ? 0 : 1); - - // Now we have the exponent and mantissa adjusted - uint32_t drop_mask = (1 << (mfmt - Wm)) - 1; - bool odd = - mantissa & (1 << (mfmt - Wm)); // if the least significant bit that is not truncated is 1 - /* - This part is doing rounding by adding mantissa part that is going to get dropped. - e.g. if the dropped part for less than 0.5 than it would round down. - if the dropped part is more than 0.5 then it would round up by rolling carry to LSB of retained - mantissa. - For the mid point when bit pattern is like this for Odd: `xy1:10000000` for Odd and - `xy0:10000000` for the Even. where `:` is delimiter for dropped v/s retained part. - For the odd case : - this will add xy1:10000000 + 000:10000000 which would roll over carry to LSB of retained - part making it RNE. - For the even case : this will add xy0:10000000 + 000:01111111 which would - round down and keep number Even - */ - mantissa += (stoch ? rng : (midpoint ? (odd ? mantissa : mantissa - 1) : mantissa)) & drop_mask; - - // Now we deal with overflow - if(f8_exponent == 0 and ((1 << mfmt) & mantissa)) - { - f8_exponent = 1; // denormal overflow to become normal, promote exponent - } - else if((1 << (mfmt + 1)) & mantissa) - { - mantissa >>= 1; - f8_exponent++; - } - - mantissa >>= (mfmt - Wm); - - // above range: quantize to maximum possible float of the same sign - // for e5m2 case, max_exp is 14, since exp = 15 is reserved for Infs and Nans - const int max_exp = (1 << We) - ((NegativeZeroNan or Wm == 3) ? 1 : 2); - if(f8_exponent > max_exp) - { - if(Clip) - return signed_max; - else - { - // https://onnx.ai/onnx/technical/float8.html#cast - if(NegativeZeroNan) - return 0x80; - else - return (Wm == 2) ? signed_inf : signed_all_ones; - } - } - - if(f8_exponent == 0 and mantissa == 0) - return NegativeZeroNan ? 0 : (sign << 7); - mantissa &= (1 << Wm) - 1; - return (sign << 7) | (f8_exponent << Wm) | mantissa; -} -// NOLINTEND - -template -__device__ constexpr T cast_from_f8(uint8_t x) -{ - // half is not supported for now - constexpr bool is_half = false; - constexpr bool is_float = true; - static_assert(is_float or is_half, "Only float are supported"); - - constexpr int weo = is_half ? 5 : 8; - constexpr int wmo = is_half ? 10 : (is_float ? 23 : 7); - // NOLINTNEXTLINE - T f_inf, f_neg_inf, f_nan, f_neg0; - - if constexpr(is_float) - { - const uint32_t if_inf = 0x7F800000; - const uint32_t if_neg_inf = 0xFF800000; - const uint32_t if_nan = 0x7F800001; - const uint32_t if_neg0 = 0x80000000; - f_inf = migraphx::bit_cast(if_inf); - f_neg_inf = migraphx::bit_cast(if_neg_inf); - f_nan = migraphx::bit_cast(if_nan); - f_neg0 = migraphx::bit_cast(if_neg0); - } - - if(x == 0) - return 0; - - uint32_t sign = x >> 7; // NOLINT - uint32_t mantissa = x & ((1 << Wm) - 1); // NOLINT - int exponent = (x & 0x7F) >> Wm; // NOLINT - if(NegativeZeroNan) - { - if(x == 0x80) - return f_nan; - } - else - { - if(x == 0x80) - return f_neg0; - if(exponent == ((1 << We) - 1) and Wm == 2) // NOLINT - return (mantissa == 0) ? (sign ? f_neg_inf : f_inf) : f_nan; - else if(Wm == 3 and (x == 0x7F or x == 0xFF)) - return f_nan; - } - typename migraphx::conditional_t retval; - - const int exp_low_cutoff = - (1 << (weo - 1)) - (1 << (We - 1)) + 1 - (NegativeZeroNan ? 1 : 0); // NOLINT - - // subnormal input - if(exponent == 0) - { - // guaranteed mantissa!=0 since cases 0x0 and 0x80 are handled above - int sh = 1 + __builtin_clz(mantissa) - (32 - Wm); - mantissa <<= sh; // NOLINT - exponent += 1 - sh; - mantissa &= ((1 << Wm) - 1); // NOLINT - } - exponent += exp_low_cutoff - 1; - mantissa <<= wmo - Wm; // NOLINT - - // subnormal output (occurs when T=half, We=5, negative_zero_nan=true) - if(exponent <= 0) - { - mantissa |= 1 << wmo; // NOLINT - mantissa >>= 1 - exponent; // NOLINT - exponent = 0; - } - - if(sizeof(T) == 2) - retval = (sign << 15) | (exponent << 10) | mantissa; // NOLINT - else - retval = (sign << 31) | (exponent << 23) | mantissa; // NOLINT - return migraphx::bit_cast(retval); -} -} // namespace impl -} // namespace fp8 -} // namespace migraphx -#if defined(__clang__) -#pragma clang diagnostic pop -#endif -#endif // MIGRAPHX_GUARD_KERNELS_FP8_IMPL_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/functional.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/functional.hpp deleted file mode 100644 index 3e9d80261..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/functional.hpp +++ /dev/null @@ -1,389 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_FUNCTIONAL_HPP -#define MIGRAPHX_GUARD_KERNELS_FUNCTIONAL_HPP - -#include - -// Similiar to decltype(auto) except it will propagate any substitution failures -// NOLINTNEXTLINE -#define MIGRAPHX_RETURNS(...) \ - ->decltype(__VA_ARGS__) { return __VA_ARGS__; } - -// Lifts an expression into a function object so it can be passed to a higher-order function -// NOLINTNEXTLINE -#define MIGRAPHX_LIFT(...) \ - [](auto&&... private_lifts_xs) MIGRAPHX_RETURNS( \ - (__VA_ARGS__)(static_cast(private_lifts_xs)...)) - -// NOLINTNEXTLINE -#define MIGRAPHX_LIFT_CLASS(name, ...) \ - struct name \ - { \ - template \ - constexpr auto operator()(PrivateLiftTs&&... private_lifts_xs) const MIGRAPHX_RETURNS( \ - (__VA_ARGS__)(static_cast(private_lifts_xs)...)) \ - } - -namespace migraphx { - -struct swallow -{ - template - constexpr swallow(Ts&&...) - { - } -}; - -template -using ignore = swallow; - -template -struct overloaded : Fs... -{ - using Fs::operator()...; - constexpr overloaded(Fs... fs) : Fs(fs)... {} -}; - -template -constexpr overloaded overload(Fs... fs) -{ - return {fs...}; -} - -namespace detail { - -template -struct eval_helper -{ - R result; - - template - constexpr eval_helper(const F& f, Ts&&... xs) : result(f(static_cast(xs)...)) - { - } -}; - -template <> -struct eval_helper -{ - int result; - template - constexpr eval_helper(const F& f, Ts&&... xs) : result((f(static_cast(xs)...), 0)) - { - } -}; - -template -struct seq -{ - using type = seq; -}; - -template -struct merge_seq; - -template -struct merge_seq, seq> : seq -{ -}; - -template -struct gens : merge_seq::type, typename gens::type> -{ -}; - -template <> -struct gens<0> : seq<> -{ -}; -template <> -struct gens<1> : seq<0> -{ -}; - -template -constexpr auto sequence_c_impl(F&& f, seq) -{ - return f(index_constant{}...); -} - -template -constexpr auto args_at(seq) -{ - return [](ignore..., auto x, auto...) { return x; }; -} - -} // namespace detail - -template -constexpr auto always(T x) -{ - return [=](auto&&...) { return x; }; -} - -template -constexpr auto sequence_c(F&& f) -{ - return detail::sequence_c_impl(f, detail::gens{}); -} - -template -constexpr auto sequence(IntegerConstant ic, F&& f) -{ - return sequence_c(f); -} - -template -constexpr auto by(F f, G g) -{ - return [=](auto... xs) { - return detail::eval_helper{g, f(xs)...}.result; - }; -} - -template -constexpr auto by(F f) -{ - return by([=](auto x) { return (f(x), 0); }, always(0)); -} - -template -constexpr void each_args(F f, Ts&&... xs) -{ - swallow{(f(static_cast(xs)), 0)...}; -} - -template -constexpr void each_args(F) -{ -} - -template -constexpr void unpack_each(F f) -{ - f(); -} - -template -constexpr void unpack_each(F f, Pack p) -{ - p([&](auto&&... xs) { each_args(f, static_cast(xs)...); }); -} - -template -constexpr void unpack_each(F f, Pack1 p1, Pack2 p2) -{ - p1([&](auto&&... xs) { - p2([&](auto&&... ys) { - each_args( - [&](auto&& p) { p(f); }, - pack_forward(static_cast(xs), static_cast(ys))...); - }); - }); -} - -template -constexpr void unpack_each(F f, Pack1 p1, Pack2 p2, Packs... packs) -{ - unpack_each( - [&](auto&& x, auto&& y) { - unpack_each( - [&](auto&&... zs) { - f(static_cast(x), - static_cast(y), - static_cast(zs)...); - }, - packs...); - }, - p1, - p2); -} - -template -constexpr void repeat_c(F&& f) -{ - sequence_c([&](auto... xs) { each_args(f, xs...); }); -} - -template -constexpr auto repeat(IntegerConstant ic, F&& f) -{ - return repeat_c(f); -} - -template -constexpr auto fold_impl(F&&, T&& x) -{ - return static_cast(x); -} - -template -constexpr auto fold_impl(F&& f, T&& x, U&& y, Ts&&... xs) -{ - return fold_impl(f, f(static_cast(x), static_cast(y)), static_cast(xs)...); -} - -template -constexpr auto fold(F f) -{ - return [=](auto&&... xs) { return fold_impl(f, static_cast(xs)...); }; -} - -template -constexpr auto compose(Fs... fs) -{ - return fold([](auto f, auto g) { - return [=](auto&&... xs) { return f(g(static_cast(xs)...)); }; - })(fs...); -} - -template -constexpr auto partial(F f) -{ - return [=](auto... xs) { - return [=](auto&&... ys) { return f(xs..., static_cast(ys)...); }; - }; -} - -template -constexpr auto pack(Ts... xs) -{ - return [=](auto f) { return f(xs...); }; -} - -template -constexpr auto pack_forward(Ts&&... xs) -{ - return [&](auto f) { return f(static_cast(xs)...); }; -} - -template -constexpr auto join(G g, F f) -{ - return f([=](auto... xs) { return g(xs...); }); -} - -template -constexpr auto join(G g, F f, Fs... fs) -{ - // return f1([=](auto x) { return f2([=](auto y) { return g(x, y); }); }); - return f([=](auto... xs) { return join([=](auto... ys) { return g(xs..., ys...); }, fs...); }); -} - -template -constexpr auto pack_compare(Compare compare, P1 p1, P2 p2) -{ - return p1([&](auto... xs) { - return p2([&](auto... ys) { - auto c = [&](auto x, auto y) -> int { - if(compare(x, y)) - return 1; - else if(compare(y, x)) - return -1; - else - return 0; - }; - return fold([](auto x, auto y) { return x ? x : y; })(c(xs, ys)..., 0); - }); - }); -} - -template -constexpr auto arg_c() -{ - return [](auto... xs) { return detail::args_at(detail::gens{})(xs...); }; -} - -template -constexpr auto arg(IntegralConstant ic) -{ - return arg_c(); -} - -template -constexpr auto make_transform(F f) -{ - return [=](auto... xs) { return [=](auto g) { return f(g, xs...); }; }; -} - -// An arg transformation takes the arguments and then a function to take the new arguments: -// transform(xs...)([](auto... ys) { ... }) -// The transform_args function takes a list of transformations and continually applies them -template -constexpr auto transform_args(F f) -{ - return f; -} - -template -constexpr auto transform_args(F f, Fs... fs) -{ - return make_transform([=](auto g, auto... xs) { - return f(xs...)([=](auto... ys) { return transform_args(fs...)(ys...)(g); }); - }); -} - -// identity transform -inline constexpr auto transform_args() -{ - return make_transform([](auto f, auto... xs) { return f(xs...); }); -} - -// Rotate the last N arguments to the first N arguments -template -constexpr auto rotate_last() -{ - return make_transform([](auto f, auto... xs) { - return sequence_c([&](auto... is) { - constexpr auto size = sizeof...(is); - return f(arg_c<(is + size - N) % size>()(xs...)...); - }); - }); -} - -inline constexpr auto rotate_last() { return rotate_last<1>(); } - -// Pack the first N arguments -template -constexpr auto pack_first() -{ - return make_transform([](auto f, auto... xs) { - return sequence_c([&](auto... is) { - return sequence_c([&](auto... js) { - return f(pack(arg_c()(xs...)...), arg_c()(xs...)...); - }); - }); - }); -} - -// Rotate the last N arguments as the first argument packed -template -constexpr auto rotate_and_pack_last() -{ - return transform_args(rotate_last(), pack_first()); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_FUNCTIONAL_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gather.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gather.hpp deleted file mode 100644 index 45f4ffcde..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gather.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_GATHER_HPP -#define MIGRAPHX_GUARD_KERNELS_GATHER_HPP - -#include -#include -#include -#include - -namespace migraphx { - -template -constexpr auto gather_shape(Input input, Indices indices) -{ - auto lengths = input.lens; - - lengths[Axis] = indices.elements(); - return make_shape(lengths, input.strides); -} - -template -__device__ void gather(Input input, Indices indices, Output output) -{ - auto ind = make_index(); - auto axis_dim_size = input.get_shape().lens[Axis]; - - constexpr auto out_comp = gather_shape(get_shape_c{}, get_shape_c{}); - - ind.global_stride(output.get_shape().elements(), [&](auto i) { - auto idx = out_comp.multi(i); - auto in_index = indices[idx[Axis]]; - - auto new_in_index = (in_index < 0) ? in_index + axis_dim_size : in_index; - - idx[Axis] = new_in_index; - - output[i] = input[idx]; - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gathernd.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gathernd.hpp deleted file mode 100644 index 325b7d34f..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gathernd.hpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_GATHERND_HPP -#define MIGRAPHX_GUARD_KERNELS_GATHERND_HPP - -#include -#include -#include -namespace migraphx { - -template -struct gathernd_settings -{ - T batch_dims{}; -}; - -template -constexpr gathernd_settings make_gathernd_settings(Ts... xs) -{ - return {xs...}; -} - -template -__device__ void gathernd(const T& data_t, const U& indices_t, const V& output_t, Settings s) -{ - auto ind = make_index(); - auto batch_dims = s.batch_dims; - auto output_shape = output_t.get_shape(); - auto indices_shape = indices_t.get_shape(); - auto data_shape = data_t.get_shape(); - - auto indices_shape_lens = indices_shape.lens; - auto data_shape_lens = data_shape.lens; - auto num_slice_dims = indices_shape_lens.back(); - size_t num_slices = - accumulate(indices_shape_lens.begin(), indices_shape_lens.end() - 1, 1, op::product{}); - size_t slice_size = accumulate(data_shape_lens.begin() + num_slice_dims + batch_dims, - data_shape_lens.end(), - 1, - op::product{}); - const size_t num_batches = - accumulate(data_shape_lens.begin(), data_shape_lens.begin() + batch_dims, 1, op::product{}); - const size_t data_batch_stride = - accumulate(data_shape_lens.begin() + batch_dims, data_shape_lens.end(), 1, op::product{}); - const auto num_slices_per_batch = num_slices / num_batches; - - ind.global_stride(output_shape.elements(), [&](auto i) { - const auto* indices_ptr = indices_t.data(); - const size_t j = i / slice_size; - const size_t batch_idx = j / num_slices_per_batch; - - auto* slice_indices = indices_ptr + (j * num_slice_dims); - size_t relative_slice_offset = 0; - for(size_t idx = 0; idx < num_slice_dims; ++idx) - { - int64_t index = slice_indices[idx]; - const size_t input_dim_idx = batch_dims + idx; - const auto input_dim = data_shape_lens[input_dim_idx]; - MIGRAPHX_ASSERT(index >= -static_cast(input_dim) and - index < static_cast(input_dim)); - if(index < 0) - index += input_dim; - size_t size_from_slice_dims = - accumulate(data_shape_lens.begin() + batch_dims + idx + 1, - data_shape_lens.begin() + batch_dims + num_slice_dims, - slice_size, - op::product{}); - relative_slice_offset += index * size_from_slice_dims; - } - - auto slice_offset = (batch_idx * data_batch_stride) + relative_slice_offset; - output_t[i] = data_t[slice_offset + i % slice_size]; - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gemm_batcher.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gemm_batcher.hpp deleted file mode 100644 index d219786c6..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gemm_batcher.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_GEMM_BATCHER_HPP -#define MIGRAPHX_GUARD_KERNELS_GEMM_BATCHER_HPP - -#include -#include -#include - -namespace migraphx { - -template -constexpr auto gemm_get_batches() -{ - constexpr auto lens = get_shape_c{}.lens; - constexpr auto strides = get_shape_c{}.strides; - constexpr auto new_lens = sequence( - lens.size() - _c<2>, [&](auto... is) { return make_const_array(_c...); }); - constexpr auto new_strides = sequence( - strides.size() - _c<2>, [&](auto... is) { return make_const_array(_c...); }); - return make_shape(new_lens, new_strides); -} - -template -constexpr auto gemm_get_matrix() -{ - constexpr auto lens = get_shape_c{}.lens; - constexpr auto strides = get_shape_c{}.strides; - constexpr auto m = lens.size() - _c<2>; - constexpr auto n = lens.size() - _c<1>; - constexpr auto new_lens = make_const_array(_c, _c); - constexpr auto new_strides = make_const_array(_c, _c); - return make_shape(new_lens, new_strides); -} - -template -constexpr auto gemm_batch_slice(Tensor t, T i) -{ - constexpr auto batch = gemm_get_batches(); - constexpr auto matrix = gemm_get_matrix(); - MIGRAPHX_ASSERT((batch.index(i) + matrix.element_space()) <= t.get_shape().element_space()); - return make_tensor_view(t.data() + batch.index(i), matrix); -} - -template -constexpr auto gemm_batch_args(index idx, BlocksPerBatch bpb, T x, Ts... xs) -{ - return [=](auto f) { - // All tensors should have the same rank - static_assert( - (true and ... and (get_shape_c{}.lens.size() == get_shape_c{}.lens.size()))); - if constexpr(get_shape_c{}.lens.size() > 2) - { - // Get the first batch since all batches should have the same number of elements - constexpr auto batch = gemm_get_batches(); - static_assert( - (true and ... and (batch.elements() == gemm_get_batches().elements()))); - idx.group_stride(bpb * batch.elements(), [&](auto gidx) { - const auto batch_idx = gidx / bpb; - f(gemm_batch_slice(x, batch_idx), gemm_batch_slice(xs, batch_idx)...); - }); - } - else - { - f(x, xs...); - } - }; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_GEMM_BATCHER_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/generic_constant.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/generic_constant.hpp deleted file mode 100644 index a1c2c9f82..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/generic_constant.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_GENERIC_CONSTANT_HPP -#define MIGRAPHX_GUARD_KERNELS_GENERIC_CONSTANT_HPP - -namespace migraphx { - -template -struct generic_constant -{ - static constexpr auto value = F{}(); - using value_type = decltype(value); - using type = generic_constant; - constexpr operator value_type() const noexcept { return value; } - constexpr value_type operator()() const noexcept { return value; } -}; - -template -constexpr generic_constant make_generic_constant(F) -{ - return {}; -} - -// NOLINTNEXTLINE -#define MIGRAPHX_MAKE_CONSTANT(x) \ - make_generic_constant([] { \ - struct fun \ - { \ - constexpr auto operator()() const { return x; } \ - }; \ - return fun{}; \ - }()) - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_GENERIC_CONSTANT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gqa_rotary_embedding.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gqa_rotary_embedding.hpp deleted file mode 100644 index 6fd1b15f3..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gqa_rotary_embedding.hpp +++ /dev/null @@ -1,178 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_ROTARY_EMBEDDING_HPP -#define MIGRAPHX_GUARD_KERNELS_ROTARY_EMBEDDING_HPP - -#include -#include -#include - -namespace migraphx { - -template -__device__ void run_rotary_embedding(Input input, - CosCache cos_cache, - SinCache sin_cache, - Output output, - PosIDs pos_ids, - Params params, - index_int idx, - bool is_query = false) -{ - const index_int batch_size = params.batch_size; - const index_int sequence_length = params.sequence_length; - const index_int n_heads = is_query ? params.num_heads : params.kv_num_heads; - const index_int head_size = params.head_size; - const index_int head_stride = params.head_stride; - const index_int seq_stride = params.seq_stride; - const index_int batch_stride = params.batch_stride; - const int position_ids_format = params.position_ids_format; - const index_int rotary_emb_dim = params.rotary_embedding_dim; - const index_int half_rotary_emb_dim = rotary_emb_dim / 2; - - const index_int loop_len = batch_size * sequence_length * n_heads; - const index_int i = idx / head_size; - const index_int ii = idx % head_size; - if(i < loop_len) - { - const index_int b = (i / n_heads) / sequence_length; - const index_int s = (i / n_heads) % sequence_length; - const index_int n = i % n_heads; - const index_int block_offset = b * batch_stride + s * seq_stride + n * head_stride; - auto input_data = input + block_offset; - auto output_data = output + block_offset; - - // Cache is (M, H/2) or (M, rotary_embedding_dim/2) - int position_id = (position_ids_format == 0) - ? static_cast(pos_ids[0]) + s - : static_cast(pos_ids[b * sequence_length + s]); - position_id = (sequence_length == 1) ? position_id : s; - - const index_int cache_offset = position_id * half_rotary_emb_dim; - auto cos_data = cos_cache + cache_offset; - auto sin_data = sin_cache + cache_offset; - - int cache_idx = 0; - double sign = 0.0; - int j = 0; - if(ii < rotary_emb_dim) - { - if(params.rotary_interleaved) - { - cache_idx = (ii / 2) % half_rotary_emb_dim; - sign = (ii % 2 == 0) ? -1.0 : 1.0; - j = (ii % 2 == 0) ? ii + 1 : ii - 1; // i - sign - } - else - { - cache_idx = ii % half_rotary_emb_dim; - sign = (ii < half_rotary_emb_dim) ? -1.0 : 1.0; - j = (ii + half_rotary_emb_dim) % rotary_emb_dim; - } - double out_data = - static_cast(input_data[ii]) * static_cast(cos_data[cache_idx]) + - sign * static_cast(input_data[j]) * - static_cast(sin_data[cache_idx]); - output_data[ii] = out_data; - } - else if(ii < head_size) - { - output_data[ii] = input_data[ii]; - } - } -} - -template -__device__ void -pack_v_into_rotary_qkv(Params params, const Input input, Output output, index_int idx) -{ - const index_int loop_len = params.batch_size * params.sequence_length * params.kv_num_heads; - auto i = idx / params.head_size; - auto ii = idx % params.head_size; - if(i < loop_len) - { - const index_int b = (i / params.kv_num_heads) / params.sequence_length; - const index_int s = (i / params.kv_num_heads) % params.sequence_length; - const index_int n = i % params.kv_num_heads; - const index_int block_offset = - b * params.batch_stride + s * params.seq_stride + n * params.head_stride; - const Input input_data = input + block_offset; - Output output_data = output + block_offset; - if(ii < params.head_size) - { - output_data[ii] = input_data[ii]; - } - } -} - -template -__device__ void gqa_rotary_embedding(Output output, - Query query, - SeqLensK seqlens_k, - CosCache cos_cache, - SinCache sin_cache, - Params params) -{ - auto ind = make_index(); - ind.global_stride(output.get_shape().elements(), [&](auto idx) { - auto q_input = query.begin(); - auto q_rotary = output.begin(); - auto k_input = q_input + params.num_heads * params.sequence_length * params.head_size; - auto k_rotary = q_rotary + params.num_heads * params.sequence_length * params.head_size; - auto v_input = k_input + params.kv_num_heads * params.sequence_length * params.head_size; - auto v_rotary = k_rotary + params.kv_num_heads * params.sequence_length * params.head_size; - auto q_chunk_size = - params.batch_size * params.num_heads * params.sequence_length * params.head_size; - auto kv_chunk_size = - params.batch_size * params.kv_num_heads * params.sequence_length * params.head_size; - if(idx < q_chunk_size) - { - run_rotary_embedding(q_input, - cos_cache.begin(), - sin_cache.begin(), - q_rotary, - seqlens_k.begin(), - params, - idx, - true); - } - else if(idx < q_chunk_size + kv_chunk_size) - { - run_rotary_embedding(k_input, - cos_cache.begin(), - sin_cache.begin(), - k_rotary, - seqlens_k.begin(), - params, - idx - q_chunk_size); - } - else if(idx < output.get_shape().elements()) - { - pack_v_into_rotary_qkv(params, v_input, v_rotary, idx - (q_chunk_size + kv_chunk_size)); - } - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gqa_softmax.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gqa_softmax.hpp deleted file mode 100644 index 27e2154b6..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/gqa_softmax.hpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_GQA_SOFTMAX_HPP -#define MIGRAPHX_GUARD_KERNELS_GQA_SOFTMAX_HPP - -#include -#include -#include - -namespace migraphx { - -template -__device__ void softmax_inplace(T score, int n, int d) -{ - for(int j = 0; j < n; ++j) - { - auto x = score + j * d; - auto y = x; - - // e^x is represented as infinity if x is large enough, like 100.f. - // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if - // one or more item are large enough. a math transform as below is - // leveraged to get a stable softmax: e^xi/(e^x1 + ...e^xn) = e^(xi - - // max) / (e^(x1 - max) + ... + e^(xn - max)) - float max = -numeric_max(); - for(int i = 0; i < d; i++) - { - if(max < x[i]) - max = x[i]; - } - for(int i = 0; i < d; i++) - { - y[i] = expf(x[i] - max); - } - - float sum = 0.0; - for(int i = 0; i < d; i++) - { - sum += x[i]; - } - - for(int i = 0; i < d; i++) - { - y[i] = x[i] / static_cast(sum); - } - } -} - -template -__device__ void calculate_softmax(AttnProbs attention_probs, // output buffer with size BxNxSxT - SeqLensK seqlens_k, // past sequence lengths tensor - Params params, - index_int idx) -{ - const index_int batch_size = params.batch_size; - const index_int sequence_length = params.sequence_length; - const index_int num_heads = params.num_heads; - const index_int present_buffer_sequence_length = params.seqlen_present_kv_cache; - - const index_int loop_len = batch_size * num_heads; - const index_int i = idx / sequence_length; - const index_int inner_i = idx % sequence_length; - if(i < loop_len) - { - const index_int batch_index = i / num_heads; - const index_int total_seqlen = seqlens_k[batch_index] + 1; - const index_int output_offset = i * sequence_length * present_buffer_sequence_length; - auto output = attention_probs + output_offset; - - const int local_window_size = params.local_window_size; - auto output_softmax = output; - index_int seq = inner_i; - if(seq < sequence_length) - { - output_softmax += seq * present_buffer_sequence_length; - auto consume = total_seqlen + local_window_size; - seq += consume; - seq -= consume; - int seq_causal_length = sequence_length == 1 ? total_seqlen : seq + 1; - if(local_window_size > 0 and seq_causal_length > local_window_size + 1) - { - for(int total_seq_id = 0; total_seq_id < seq_causal_length - local_window_size - 1; - total_seq_id++) - { - output_softmax[total_seq_id] = 0.f; - } - softmax_inplace(output_softmax + seq_causal_length - local_window_size - 1, - 1, - local_window_size + 1); - } - else - { - softmax_inplace(output_softmax, 1, seq_causal_length); - } - for(int total_seq_id = seq_causal_length; total_seq_id < total_seqlen; total_seq_id++) - { - output_softmax[total_seq_id] = 0.f; - } - } - } -} - -template -__device__ void -gqa_softmax(Output output, Input, PresentKey, Probs, SeqLensK seqlens_k, Params params) -{ - const index_int elements = params.batch_size * params.num_heads * params.sequence_length; - auto ind = make_index(); - ind.global_stride(elements, [&](auto idx) { - calculate_softmax(output.begin(), seqlens_k.begin(), params, idx); - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/group_query_attention.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/group_query_attention.hpp deleted file mode 100644 index dbb60e7bd..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/group_query_attention.hpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_GROUP_QUERY_ATTENTION_HPP -#define MIGRAPHX_GUARD_KERNELS_GROUP_QUERY_ATTENTION_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { - -template -struct gqa_parameters -{ - T1 scale; - T2 batch_size; // Batch size used by input - T3 sequence_length; // Sequence length used by input - T4 hidden_size; // Hidden size used by input - T5 head_size; // Head size - T6 rotary_embedding_dim; // Rotary embedding dimension. - T7 num_heads; // num_heads = hidden_size / head_size - T8 max_sequence_length; // Sequence length used by cos/sin cache - T9 head_stride; // Head stride - T10 seq_stride; // Sequence stride - T11 batch_stride; // Batch stride - T12 position_ids_format; // Format of position ids - 0 is (1), 1 is (batch_size, - // sequence_length) - T13 seqlen_present_kv_cache; // Sequence length of present kv-cache (4096 when using - // shared buffer) - T14 do_rotary; // Whether to use rotary position embedding. Default value is 0. - T15 kv_num_heads; // Number of attention heads for k and v - T16 local_window_size; // left_window_size for local attention. Default value is -1 meaning - // unused. - T17 rotary_interleaved; // Rotate using interleaved pattern. Default value is 0 (False). - T18 past_present_share_buffer; // Whether to use same buffer for KV-cache inputs and outputs -}; - -template -__device__ gqa_parameters make_gqa_parameters(Ts... ts) -{ - return {ts...}; -} - -struct naive_gemm -{ - index_int max_m; - index_int max_n; - index_int max_k; - index_int lda; - index_int ldb; - index_int ldc; - bool b_transpose; - float alpha; - float beta; - - template - __device__ void compute(C cmat, const A amat, const B bmat, const index_int idx) - { - auto m = idx / max_n; - auto n = idx % max_n; - auto index = [&](auto x, auto y, auto z) { return y + (x * z); }; - - if(m < max_m) - { - if(n < max_n) - { - double s = 0.0; - for(int k = 0; k < max_k; ++k) - { - auto a_i = index(m, k, lda); - auto b_i = b_transpose ? index(n, k, ldb) : index(k, n, ldb); - s += static_cast(amat[a_i]) * static_cast(bmat[b_i]); - } - auto c_i = index(m, n, ldc); - cmat[c_i] = static_cast(alpha) * s + cmat[c_i] * static_cast(beta); - } - } - } -}; - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/hip.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/hip.hpp deleted file mode 100644 index 8ddc7ad0e..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/hip.hpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_HIP_HPP -#define MIGRAPHX_GUARD_KERNELS_HIP_HPP - -#ifndef MIGRAPHX_USE_HIPRTC -#include -#include -#include -#endif - -#endif // MIGRAPHX_GUARD_KERNELS_HIP_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/index.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/index.hpp deleted file mode 100644 index 9c43f5d3b..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/index.hpp +++ /dev/null @@ -1,309 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_INDEX_HPP -#define MIGRAPHX_GUARD_KERNELS_INDEX_HPP - -#include -#include -#include -#include -#include -#include - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreserved-identifier" -extern "C" __device__ size_t __ockl_get_enqueued_local_size(uint); // NOLINT -extern "C" __device__ size_t __ockl_get_local_size(uint); // NOLINT -#pragma clang diagnostic pop -#endif - -namespace migraphx { - -#if defined(MIGRAPHX_NGLOBAL) && defined(MIGRAPHX_NLOCAL) -#define MIGRAPHX_NGROUP ((MIGRAPHX_NGLOBAL + MIGRAPHX_NLOCAL - 1) / MIGRAPHX_NLOCAL) -#endif - -inline __device__ __attribute__((const)) index_int compute_global_size() -{ -#ifdef MIGRAPHX_NGLOBAL - return MIGRAPHX_NGLOBAL; -#else - // This actually works even when global is not divisible by local size. - // This doesnt actually do a multiplication. Instead it calls a device - // function to get the global size, which is why it works. - return blockDim.x * gridDim.x; // NOLINT -#endif -} - -#ifdef MIGRAPHX_NGROUP -// If global is divisible by local then local can be a const -#if(MIGRAPHX_NGLOBAL % MIGRAPHX_NLOCAL == 0) || (MIGRAPHX_NGROUP == 1) -#define MIGRAPHX_HAS_CONST_LOCAL 1 -#endif -#endif - -inline __device__ __attribute__((const)) index_int compute_local_size() -{ -#ifdef MIGRAPHX_HAS_CONST_LOCAL - return MIGRAPHX_NLOCAL; -#else - // Returns block size. For the non-uniform block it returns the size of the non-uniform block. - return __ockl_get_local_size(0); // NOLINT -#endif -} - -inline __device__ __attribute__((const)) index_int compute_max_local_size() -{ -#ifdef MIGRAPHX_LOCAL - return MIGRAPHX_NLOCAL; -#else - // Returns the block size. When workgrop has non-uniform block, this returns size of the uniform - // block. - return __ockl_get_enqueued_local_size(0); // NOLINT -#endif -} - -struct index -{ - index_int global = 0; - index_int local = 0; - index_int group = 0; - -#ifdef MIGRAPHX_NGLOBAL - constexpr index_constant nglobal() const - { - static_assert(MIGRAPHX_NGLOBAL > 0, "Global size must be greater than 0"); - return {}; - } -#else - __device__ index_int nglobal() const - { - MIGRAPHX_ASSERT(compute_global_size() > 0); - return compute_global_size(); // NOLINT - } -#endif - -#ifdef MIGRAPHX_HAS_CONST_LOCAL - constexpr index_constant nlocal() const - { - static_assert(MIGRAPHX_NLOCAL > 0, "Local size must be greater than 0"); - return {}; - } -#else - __device__ index_int nlocal() const - { -#ifdef MIGRAPHX_NGROUP - static_assert((MIGRAPHX_NGLOBAL % MIGRAPHX_NLOCAL != 0) and (MIGRAPHX_NGROUP > 1), - "Local size should be const"); -#endif - MIGRAPHX_ASSERT(compute_local_size() > 0); - return compute_local_size(); // NOLINT - } -#endif - -#ifdef MIGRAPHX_NLOCAL - constexpr index_constant max_nlocal() const { return {}; } -#else - __device__ index_int max_nlocal() const - { - MIGRAPHX_ASSERT(compute_max_local_size() > 0); - return compute_max_local_size(); - } -#endif - - constexpr auto ngroup() const { return nglobal() / max_nlocal(); } - - template - constexpr index_constant nlocal_subwave() const - { - return {}; - } - template - constexpr auto local_subwave() const - { -#ifdef MIGRAPHX_HAS_CONST_LOCAL - if constexpr(decltype(nlocal()){} == SubWaveSize) - return local; -#endif - return local % nlocal_subwave(); - } - template - constexpr auto nwave() const - { - return max_nlocal() / nlocal_subwave(); - } - - constexpr index_constant nlocal_wave() const { return {}; } - constexpr auto local_wave() const { return local % nlocal_wave(); } - constexpr auto nwave() const { return max_nlocal() / nlocal_wave(); } - constexpr auto wave() const { return local / nlocal_wave(); } - - template - static constexpr auto max_stride_iterations(N n, Stride stride) - { - return (n - _c<1>) / stride + _c<1>; - } - - template - constexpr auto max_global_stride_iterations(N n) const - { - return max_stride_iterations(n, nglobal()); - } - - template - constexpr auto max_local_stride_iterations(N n) const - { - return max_stride_iterations(n, nlocal()); - } - - template - constexpr auto max_local_wave_stride_iterations(N n) const - { - return max_stride_iterations(n, nlocal_wave()); - } - - template - constexpr auto max_local_subwave_stride_iterations(N n) const - { - return max_stride_iterations(n, nlocal_subwave()); - } - - template - static constexpr auto invoke_loop(F f, I i, D d) -> decltype(f(i, d)) - { - return f(i, d); - } - - template - static constexpr auto invoke_loop(F f, I i, D) -> decltype(f(i)) - { - return f(i); - } - - template - static constexpr void for_stride_loop_unroll(index_int start, N n, Stride stride, F f) - { - sequence(max_stride_iterations(n, stride), [&](auto... ks) { - fold([&](auto d, auto k) { - auto i = start + stride * k; - if(i < n) - invoke_loop(f, i, d); - return d + _c<1>; - })(_c<0>, ks...); - }); - } - - template - static constexpr void for_stride_loop(index_int start, N n, Stride stride, F f) - { - index_int k = 0; - for(index_int i = start; i < n; i += stride) - { - invoke_loop(f, i, k); - k++; - } - } - - template - static constexpr void for_stride(index_int start, N n, Stride stride, F f) - { - MIGRAPHX_ASSERT(start < stride); - if constexpr(not is_integral{} and not is_integral{}) - { - if constexpr(max_stride_iterations(n, stride) == 1) - { - if constexpr(stride > n) - { - if(start < n) - invoke_loop(f, start, _c<0>); - } - else - { - invoke_loop(f, start, _c<0>); - } - } - else if constexpr(Unroll) - { - MIGRAPHX_STATIC_ASSERT_FOR(max_stride_iterations(n, stride) < 256) - { - for_stride_loop_unroll(start, n, stride, f); - } - } - else - { - for_stride_loop(start, n, stride, f); - } - } - else - { - for_stride_loop(start, n, stride, f); - } - } - - template - __device__ void global_stride(N n, F f) const - { - for_stride(global, n, nglobal(), f); - } - - template - __device__ void local_stride(N n, F f) const - { - for_stride(local, n, nlocal(), f); - } - - template - __device__ void group_stride(N n, F f) const - { - for_stride(group, n, ngroup(), f); - } - - template - __device__ void local_subwave_stride(N n, F f) const - { - for_stride(local_subwave(), n, nlocal_subwave(), f); - } - - template - __device__ void local_wave_stride(N n, F f) const - { - for_stride(local_wave(), n, nlocal_wave(), f); - } -}; - -#ifdef MIGRAPHX_NLOCAL -#define MIGRAPHX_GLOBAL \ - __global__ __attribute__((amdgpu_flat_work_group_size(MIGRAPHX_NLOCAL, MIGRAPHX_NLOCAL))) -#else -#define MIGRAPHX_GLOBAL __global__ -#endif -inline __device__ __attribute__((const)) index make_index() -{ - return index{ - blockIdx.x * compute_max_local_size() + threadIdx.x, threadIdx.x, blockIdx.x}; // NOLINT -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_INDEX_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/integral_constant.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/integral_constant.hpp deleted file mode 100644 index 63807ff78..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/integral_constant.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_INTEGRAL_CONSTANT_HPP -#define MIGRAPHX_GUARD_KERNELS_INTEGRAL_CONSTANT_HPP - -#include - -namespace migraphx { - -template -struct integral_constant -{ - static constexpr T value = V; - using value_type = T; - using type = integral_constant; - constexpr operator value_type() const noexcept { return value; } - constexpr value_type operator()() const noexcept { return value; } - static constexpr type to() { return {}; } -}; - -// NOLINTNEXTLINE -#define MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(op) \ - template \ - constexpr inline integral_constant operator op( \ - integral_constant, integral_constant) noexcept \ - { \ - return {}; \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_INTEGRAL_CONSTANT_UNARY_OP(op) \ - template \ - constexpr inline integral_constant operator op( \ - integral_constant) noexcept \ - { \ - return {}; \ - } - -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(+) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(-) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(*) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(/) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(%) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(>>) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(<<) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(>) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(<) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(<=) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(>=) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(==) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(!=) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(&) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(^) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(|) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(and) -MIGRAPHX_INTEGRAL_CONSTANT_BINARY_OP(or) - -MIGRAPHX_INTEGRAL_CONSTANT_UNARY_OP(not ) -MIGRAPHX_INTEGRAL_CONSTANT_UNARY_OP(~) -MIGRAPHX_INTEGRAL_CONSTANT_UNARY_OP(+) -MIGRAPHX_INTEGRAL_CONSTANT_UNARY_OP(-) - -template -using bool_constant = integral_constant; - -using true_type = bool_constant; -using false_type = bool_constant; - -template -using index_constant = integral_constant; - -template -static constexpr auto _c = integral_constant{}; // NOLINT - -template -constexpr auto return_c(F f) -{ - return _c; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_INTEGRAL_CONSTANT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/iota_iterator.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/iota_iterator.hpp deleted file mode 100644 index c04522778..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/iota_iterator.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_IOTA_ITERATOR_HPP -#define MIGRAPHX_GUARD_KERNELS_IOTA_ITERATOR_HPP - -#include -#include -#include - -namespace migraphx { - -template -struct basic_iota_iterator -{ - Iterator index; - F f; - - using difference_type = diff_int; - using reference = decltype(f(declval())); - using value_type = remove_reference_t; - using pointer = add_pointer_t; - - constexpr basic_iota_iterator& operator+=(diff_int n) - { - index += n; - return *this; - } - - constexpr basic_iota_iterator& operator-=(diff_int n) - { - index -= n; - return *this; - } - - constexpr basic_iota_iterator& operator++() - { - index++; - return *this; - } - - constexpr basic_iota_iterator& operator--() - { - index--; - return *this; - } - - constexpr basic_iota_iterator operator++(int) // NOLINT - { - basic_iota_iterator it = *this; - index++; - return it; - } - - constexpr basic_iota_iterator operator--(int) // NOLINT - { - basic_iota_iterator it = *this; - index--; - return it; - } - // TODO: operator-> - constexpr reference operator*() const { return f(index); } - - constexpr reference operator[](MIGRAPHX_CAPTURE_SOURCE_LOCATION(index_int) x) const - { - return f(capture_transform(x, [&](auto y) { return index + y; })); - } -}; - -template -constexpr basic_iota_iterator make_basic_iota_iterator(T x, F f) -{ - return basic_iota_iterator{x, f}; -} - -template -constexpr basic_iota_iterator operator+(basic_iota_iterator x, diff_int y) -{ - return x += y; -} - -template -constexpr basic_iota_iterator operator+(diff_int x, basic_iota_iterator y) -{ - return y + x; -} - -template -constexpr diff_int operator-(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index - y.index; -} - -template -constexpr basic_iota_iterator operator-(basic_iota_iterator x, diff_int y) -{ - return x -= y; -} - -template -constexpr bool operator==(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index == y.index; -} - -template -constexpr bool operator!=(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index != y.index; -} - -template -constexpr bool operator<(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index < y.index; -} - -template -constexpr bool operator>(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index > y.index; -} - -template -constexpr bool operator>=(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index >= y.index; -} - -template -constexpr bool operator<=(basic_iota_iterator x, basic_iota_iterator y) -{ - return x.index <= y.index; -} - -struct defaul_iota_iterator -{ - template - constexpr auto operator()(T x) const - { - return x; - } -}; - -using iota_iterator = basic_iota_iterator; - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_IOTA_ITERATOR_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/layernorm.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/layernorm.hpp deleted file mode 100644 index c64ab5531..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/layernorm.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_LAYERNORM_HPP -#define MIGRAPHX_GUARD_KERNELS_LAYERNORM_HPP -#include -#include -#include -#include - -namespace migraphx { - -template -struct acc_type -{ - using type = float; -}; - -template <> -struct acc_type -{ - using type = double; -}; - -template -constexpr auto vec_reduce(const array& a, Op op) -{ - return a.apply([&](auto x) { return vec_reduce(x, op); }); -} - -template -__device__ void generic_binary_layernorm( - F compute, BinOp op, float eps, Output output, Input1 input1, Input2 input2, Inputs... inputs) -{ - using block = reduce::auto_block()>; - using reduce_output = reduce::with_axis; - - block::template run([&](auto, auto r) { - using value_type = typename Input1::type; - using vec_value_type = typename acc_type>::type; - - auto input = r.inner([&](auto x1, auto x2) { - return migraphx::convert(op(x1, x2)); - })(input1, input2); - - constexpr auto relements = r.template elements(); - constexpr auto relements_r = vec_value_type{1.0 / relements}; - auto relements_rsqrt = sqrt(relements_r); - - auto means = r.reduce(op::sum{}, make_array(0, 0), [&](auto x) { - auto x_out = x * relements_r; - // dividing x by sqrt(relements) before squaring allows computing - // higher values before overflow in low precision - auto x2_sqrt = x * relements_rsqrt; - return make_array(x_out, x2_sqrt * x2_sqrt); - })(input); - - auto mean_x = means[0]; - auto mean_x2 = means[1]; - auto variance = mean_x2 - (mean_x * mean_x); - vec_value_type eps_val = implicit_conversion(eps); - auto rsqrt_val = rsqrt(variance + eps_val); - - r.inner([&](auto& y, auto x, auto... xs) { - y = compute(migraphx::convert>((x - mean_x) * rsqrt_val), xs...); - })(output, input, inputs...); - }); -} - -template -__device__ void layernorm(F compute, float eps, Output output, Input input, Inputs... inputs) -{ - generic_binary_layernorm( - compute, [](auto x, auto) { return x; }, eps, output, input, input, inputs...); -} - -template -__device__ void -add_layernorm(F compute, float eps, Output output, Input1 input1, Input2 input2, Inputs... inputs) -{ - generic_binary_layernorm( - compute, [](auto x1, auto x2) { return x1 + x2; }, eps, output, input1, input2, inputs...); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_LAYERNORM_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/math.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/math.hpp deleted file mode 100644 index 5052d6611..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/math.hpp +++ /dev/null @@ -1,300 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_MATH_HPP -#define MIGRAPHX_GUARD_KERNELS_MATH_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -namespace math { - -template -constexpr auto as_float(T x) -{ - if constexpr(is_integral{}) - return x; - else - return float(x); -} - -template -constexpr auto to_native(T x) -{ - return x; -} - -constexpr migraphx::half to_native(__half x) { return bit_cast(x); } - -template ())> -__device__ auto wrap(F f, T x, Ts... xs) -{ - if constexpr(is_integral{}) - { - return wrap(f, double(x), double(xs)...); - } - else if constexpr(is_callable{}) - { - return to_native(f(x, xs...)); - } - else - { - T result = f(as_float(x), as_float(xs)...); - return result; - } -} - -} // namespace math - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_LIFT_IMPL(type, ...) \ - [](type x, auto... xs) MIGRAPHX_RETURNS((__VA_ARGS__)(x, xs...)) - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_LIFT(...) MIGRAPHX_DEVICE_MATH_LIFT_IMPL(__VA_ARGS__) - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_PARSE(x) x, - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_EACH(f) MIGRAPHX_DEVICE_MATH_LIFT(MIGRAPHX_DEVICE_MATH_PARSE f) - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_WRAP(name, ...) \ - namespace math { \ - inline static constexpr auto wrap_##name = \ - overload(MIGRAPHX_PP_TRANSFORM_ARGS(MIGRAPHX_DEVICE_MATH_EACH, __VA_ARGS__)); \ - } \ - template \ - auto __device__ name(Ts... xs) MIGRAPHX_RETURNS(math::wrap(math::wrap_##name, xs...)) - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH(name, fname) \ - template ())> \ - auto __device__ name(Ts... xs) MIGRAPHX_RETURNS(fname(xs...)) - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_VEC(name) \ - template ())> \ - auto __device__ name(Ts... xs) \ - { \ - return vec_transform(xs...)([](auto... ys) { return name(ys...); }); \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_FOR(type, name, fname) \ - template ())> \ - auto __device__ name(type x, Ts... xs) -> type \ - { \ - return fname(x, xs...); \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_BINARY_FOR(type, name, fname) \ - inline auto __device__ name(type x, type y) -> type { return fname(x, y); } - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_HALF2(name, fname) \ - template \ - auto __device__ name(migraphx::vec x, Ts... xs) \ - MIGRAPHX_RETURNS(migraphx::vec{fname(x, xs...)}); \ - template 2))> \ - auto __device__ name(migraphx::vec x, Ts... xs) \ - { \ - return vec_packed_transform<2>(x, xs...)( \ - [](auto... ys) -> migraphx::vec { return fname(ys...); }); \ - } - -// Template with two overloads for math functions, one for half2 type and one for more generic -// vectorization where N is 4 or another even number. -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_MATH_VEC2(type, name, fname) \ - template \ - auto __device__ name(migraphx::vec x, Ts... xs) \ - MIGRAPHX_RETURNS(migraphx::vec{fname(x, xs...)}); \ - template 2))> \ - auto __device__ name(migraphx::vec x, Ts... xs) \ - { \ - return vec_packed_transform<2>(x, xs...)( \ - [](auto... ys) -> migraphx::vec { return fname(ys...); }); \ - } - -MIGRAPHX_DEVICE_MATH_WRAP(acos, (double)::acos, (float)::acosf); -MIGRAPHX_DEVICE_MATH_WRAP(acosh, (double)::acosh, (float)::acoshf); -MIGRAPHX_DEVICE_MATH_WRAP(asin, (double)::asin, (float)::asinf); -MIGRAPHX_DEVICE_MATH_WRAP(asinh, (double)::asinh, (float)::asinh); -MIGRAPHX_DEVICE_MATH_WRAP(atan, (double)::atan, (float)::atan); -MIGRAPHX_DEVICE_MATH_WRAP(atanh, (double)::atanh, (float)::atanh); -MIGRAPHX_DEVICE_MATH_WRAP(ceil, (double)::ceil, (float)::ceilf, (half)::hceil); -MIGRAPHX_DEVICE_MATH_WRAP(cos, (double)::cos, (float)::cosf, (half)::hcos); -MIGRAPHX_DEVICE_MATH_WRAP(cosh, (double)::cosh, (float)::coshf); -MIGRAPHX_DEVICE_MATH_WRAP(erf, (double)::erf, (float)::erff); -MIGRAPHX_DEVICE_MATH_WRAP(exp, (double)::exp, (float)::expf, (half)::hexp); -MIGRAPHX_DEVICE_MATH_WRAP(floor, (double)::floor, (float)::floorf, (half)::hfloor); -MIGRAPHX_DEVICE_MATH_WRAP(isnan, (double)::isnan, (float)::isnan, (half)::__hisnan); -MIGRAPHX_DEVICE_MATH_WRAP(isinf, (double)::isinf, (float)::isinf, (half)::__hisinf); -MIGRAPHX_DEVICE_MATH_WRAP(log, (double)::log, (float)::logf, (half)::hlog); -MIGRAPHX_DEVICE_MATH_WRAP(log2, (double)::log2, (float)::log2f, (half)::hlog2); -MIGRAPHX_DEVICE_MATH_WRAP(nearbyint, (double)::nearbyint, (float)::nearbyintf); -MIGRAPHX_DEVICE_MATH_WRAP(pow, (double)::pow, (float)::powf); -MIGRAPHX_DEVICE_MATH_WRAP(remainder, (double)::remainder, (float)::remainderf); -MIGRAPHX_DEVICE_MATH_WRAP(round, (double)::round, (float)::roundf); -MIGRAPHX_DEVICE_MATH_WRAP(rsqrt, (double)::rsqrt, (float)::rsqrtf, (half)::hrsqrt); -MIGRAPHX_DEVICE_MATH_WRAP(sin, (double)::sin, (float)::sinf, (half)::hsin); -MIGRAPHX_DEVICE_MATH_WRAP(sinh, (double)::sinh, (float)::sinhf); -MIGRAPHX_DEVICE_MATH_WRAP(sqrt, (double)::sqrt, (float)::sqrtf, (half)::hsqrt); -MIGRAPHX_DEVICE_MATH_WRAP(tan, (double)::tan, (float)::tanf); -MIGRAPHX_DEVICE_MATH_WRAP(tanh, (double)::tanh, (float)::tanhf); -MIGRAPHX_DEVICE_MATH_WRAP(fmod, (double)::fmod, (float)::fmodf); - -template -constexpr auto where(bool cond, const T& a, const U& b) -{ - return cond ? a : b; -} - -MIGRAPHX_DEVICE_MATH_FOR(float, abs, ::abs) -MIGRAPHX_DEVICE_MATH_FOR(double, abs, ::abs) -MIGRAPHX_DEVICE_MATH_FOR(migraphx::half, abs, ::__habs) -MIGRAPHX_DEVICE_MATH_FOR(migraphx::bf16, abs, ::fabsf) -MIGRAPHX_DEVICE_MATH_BINARY_FOR(float, max, ::fmaxf) -MIGRAPHX_DEVICE_MATH_BINARY_FOR(float, min, ::fminf) -MIGRAPHX_DEVICE_MATH_BINARY_FOR(double, max, ::max) -MIGRAPHX_DEVICE_MATH_BINARY_FOR(double, min, ::min) -MIGRAPHX_DEVICE_MATH_BINARY_FOR(migraphx::half, max, ::__hmax) -MIGRAPHX_DEVICE_MATH_BINARY_FOR(migraphx::half, min, ::__hmin) - -template () and is_integral{})> -constexpr auto abs(const T& a) -{ - return where(a < 0, -a, a); -} - -template ())> -constexpr auto max(const T& a, const T& b) -{ - return where(a < b, b, a); -} - -template ())> -constexpr auto min(const T& a, const T& b) -{ - return where(a < b, a, b); -} - -template {} and not is_any_vec())> -constexpr auto max(const T& a, const U& b) -{ - return max>(a, b); -} - -template {} and not is_any_vec())> -constexpr auto min(const T& a, const U& b) -{ - return min>(a, b); -} - -template ())> -constexpr T mod(const T& a, const T& b) -{ - if constexpr(is_integral{}) - // onnx mod operator requires numpy style modulus - return ((a % b) + b) % b; - return static_cast(fmod(remainder(a, b) + b, b)); -} - -template {} and not is_any_vec())> -constexpr auto mod(const T& a, const U& b) -{ - return mod>(a, b); -} - -MIGRAPHX_DEVICE_MATH_VEC(abs) -MIGRAPHX_DEVICE_MATH_VEC(acos) -MIGRAPHX_DEVICE_MATH_VEC(acosh) -MIGRAPHX_DEVICE_MATH_VEC(asin) -MIGRAPHX_DEVICE_MATH_VEC(asinh) -MIGRAPHX_DEVICE_MATH_VEC(atan) -MIGRAPHX_DEVICE_MATH_VEC(atanh) -MIGRAPHX_DEVICE_MATH_VEC(ceil) -MIGRAPHX_DEVICE_MATH_VEC(cos) -MIGRAPHX_DEVICE_MATH_VEC(cosh) -MIGRAPHX_DEVICE_MATH_VEC(erf) -MIGRAPHX_DEVICE_MATH_VEC(exp) -MIGRAPHX_DEVICE_MATH_VEC(floor) -MIGRAPHX_DEVICE_MATH_VEC(fmod) -MIGRAPHX_DEVICE_MATH_VEC(isinf) -MIGRAPHX_DEVICE_MATH_VEC(isnan) -MIGRAPHX_DEVICE_MATH_VEC(log) -MIGRAPHX_DEVICE_MATH_VEC(log2) -MIGRAPHX_DEVICE_MATH_VEC(max) -MIGRAPHX_DEVICE_MATH_VEC(min) -MIGRAPHX_DEVICE_MATH_VEC(mod) -MIGRAPHX_DEVICE_MATH_VEC(nearbyint) -MIGRAPHX_DEVICE_MATH_VEC(pow) -MIGRAPHX_DEVICE_MATH_VEC(remainder) -MIGRAPHX_DEVICE_MATH_VEC(round) -MIGRAPHX_DEVICE_MATH_VEC(rsqrt) -MIGRAPHX_DEVICE_MATH_VEC(sin) -MIGRAPHX_DEVICE_MATH_VEC(sinh) -MIGRAPHX_DEVICE_MATH_VEC(sqrt) -MIGRAPHX_DEVICE_MATH_VEC(tan) -MIGRAPHX_DEVICE_MATH_VEC(tanh) -MIGRAPHX_DEVICE_MATH_VEC(where) - -// Map math functions to hip half2 functions -// The half2 type is defined in include/hip/amd_detail/hip_fp16_gcc.h and is 2 16-bit floats -// packed into a 32-bit number. See include/hip/amd_detail/hip_fp16_math_fwd.h for the HIP names -// Most but not all of these math ops have operators of the same names. -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, abs, ::__habs2) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, ceil, ::h2ceil) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, cos, ::h2cos) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, exp, ::h2exp) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, exp10, ::h2exp10) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, exp2, ::h2exp2) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, floor, ::h2floor) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, isinf, ::__hisinf2) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, isnan, ::__hisnan2) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, log, ::h2log) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, log10, ::h2log10) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, log2, ::h2log2) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, rsqrt, ::h2rsqrt) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, sin, ::h2sin) -MIGRAPHX_DEVICE_MATH_VEC2(migraphx::half, sqrt, ::h2sqrt) - -template -constexpr auto convert(U v) -{ - return vec_transform(v)([](auto x) -> T { return static_cast(x); }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_MATH_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/operators.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/operators.hpp deleted file mode 100644 index 35f9c920e..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/operators.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_OPERATORS_HPP -#define MIGRAPHX_GUARD_KERNELS_OPERATORS_HPP - -#include -#include - -namespace migraphx { - -template -struct equality_comparable -{ - template - friend constexpr auto operator!=(const T& x, const U& y) MIGRAPHX_RETURNS(not(x == y)); - template {} and is_same{})> - friend constexpr auto operator!=(const U& x, const V& y) MIGRAPHX_RETURNS(not(x == y)); -}; - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_OPERATORS_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ops.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ops.hpp deleted file mode 100644 index be1ece0f9..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ops.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_OPS_HPP -#define MIGRAPHX_GUARD_KERNELS_OPS_HPP - -#include - -namespace migraphx { -namespace op { - -struct sum -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return x + y; - } -}; - -struct product -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return x * y; - } -}; - -struct id -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x) const - { - return x; - } -}; - -template -struct convert_to -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(U x) const - { - return convert(x); - } -}; - -template -struct mean -{ - template - MIGRAPHX_DEVICE_CONSTEXPR T operator()(T x) const - { - using type = vec_type; - if constexpr(is_floating_point{}) - { - constexpr type d = 1.0 / N; - return x * d; - } - else - { - return x / static_cast(N); - } - } -}; - -struct max -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return migraphx::max(x, y); - } -}; - -struct min -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - return migraphx::min(x, y); - } -}; - -struct logical_and -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - if(static_cast(x) and static_cast(y)) - return static_cast(1); - return static_cast(0); - } -}; - -struct logical_or -{ - template - MIGRAPHX_DEVICE_CONSTEXPR auto operator()(T x, U y) const - { - if(static_cast(x) or static_cast(y)) - return static_cast(1); - return static_cast(0); - } -}; -} // namespace op - -// NOLINTNEXTLINE -#define MIGRAPHX_OPS_DEFINE_COMMON_TYPE(T) \ - template \ - struct common_type \ - { \ - using type = U; \ - }; \ - template \ - struct common_type \ - { \ - using type = U; \ - }; - -struct lowest -{ - template - constexpr operator T() const - { - return numeric_lowest>(); - } -}; -MIGRAPHX_OPS_DEFINE_COMMON_TYPE(lowest) - -struct highest -{ - template - constexpr operator T() const - { - return numeric_max>(); - } -}; - -MIGRAPHX_OPS_DEFINE_COMMON_TYPE(highest) - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_OPS_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pad.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pad.hpp deleted file mode 100644 index 38d8be2de..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pad.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_PAD_HPP -#define MIGRAPHX_GUARD_KERNELS_PAD_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { - -template -__device__ void pad(const index& idx, - const Offsets& offsets, - const Input& input, - Output& output, - const PadVal& pad_val) -{ - auto output_shape = output.get_shape(); - idx.global_stride(output_shape.elements(), [&](auto i) { - // 1. get current multi-index for output - // 2. get the size of the input to determine input boundaries - // 3. compute the corresponding multi-index for input by accounting for offsets - // 4. if current multi-index is within offsets or input's new multi-index is out of bounds, - // use pad value instead of input's value - auto multi = output_shape.multi(i); - auto input_bounds = input.get_shape().lens; - auto input_idx = multi - offsets; - auto range_multi = range(multi.size()); - - if(any_of(range_multi.begin(), range_multi.end(), [&](auto j) { - return multi[j] < offsets[j] or input_idx[j] >= input_bounds[j]; - })) - output[multi] = implicit_conversion(pad_val); - else - output[multi] = implicit_conversion(input[input_idx]); - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/permutation.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/permutation.hpp deleted file mode 100644 index 970484d6f..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/permutation.hpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_PERMUTATION_HPP -#define MIGRAPHX_GUARD_KERNELS_PERMUTATION_HPP - -#include -#include -#include - -namespace migraphx { - -template -constexpr auto reorder_dims(const Array1& dims, const Array2& permutation) -{ - return generate_array( - dims.size(), [&](auto i) { return dims[permutation[i]]; }); -} - -template -constexpr auto reorder_dims(integral_const_array, integral_const_array) -{ - return return_array_c([] { - constexpr integral_const_array dims{}; - constexpr integral_const_array permutation{}; - return reorder_dims(dims.base(), permutation.base()); - }); -} - -template -constexpr auto invert_permutation(const Array& permutation) -{ - return reorder_dims(transform_i(permutation, [](auto, auto i) { return i; }), permutation); -} - -template -struct find_permutation_impl -{ - static constexpr auto compute() - { - return return_array_c([] { - constexpr Shape s{}; - typename Shape::index_array perm; - iota(perm.begin(), perm.end(), 0); - if constexpr(s.transposed() or s.broadcasted()) - { - stable_sort( - perm.begin(), - perm.end(), - by([&](auto x) { return make_tuple(s.strides[x], s.lens[x]); }, greater{})); - } - return perm; - }); - } - using type = decltype(compute()); -}; - -template -constexpr auto find_permutation(Shape) -{ - return typename find_permutation_impl::type{}; -} - -template -constexpr auto find_permutation(Shape1, Shape2) -{ - return return_array_c([] { - constexpr Shape1 s1{}; - constexpr Shape2 s2{}; - auto perm1 = find_permutation(s1).base(); - auto perm2 = find_permutation(s2).base(); - if(perm1 == perm2) - return perm1; - if(s1.standard()) - return perm1; - if(s2.standard()) - return perm2; - if(s1.packed()) - return perm1; - if(s2.packed()) - return perm2; - return perm1; - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_PERMUTATION_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pointwise.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pointwise.hpp deleted file mode 100644 index d97355a1d..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pointwise.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_POINTWISE_HPP -#define MIGRAPHX_GUARD_KERNELS_POINTWISE_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -template -__device__ void pointwise_tensor(Stride stride, F f, Output out, T x, Ts... xs) -{ - stride(x.get_shape().elements(), [&](auto i) { - auto r = f(x[i], xs[i]...); - out([&](auto... outs) { - r([&](auto... rs) { - static_assert(sizeof...(outs) == sizeof...(rs)); - swallow{(outs[i] = implicit_conversion(rs))...}; - }); - }); - }); -} - -template -__device__ auto pointwise(index idx, Transforms... transforms) -{ - return [=](auto f, auto*... ps) { - auto t = transform_args(make_tensors(), transforms..., rotate_and_pack_last()); - t(ps...)([&](auto... xs) { pointwise_tensor(tile_stride(idx), f, xs...); }); - }; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_POINTWISE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pooling.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pooling.hpp deleted file mode 100644 index 76bb7c3cb..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pooling.hpp +++ /dev/null @@ -1,233 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_POOLING_HPP -#define MIGRAPHX_GUARD_KERNELS_POOLING_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -template -struct pool_op -{ - template - MIGRAPHX_DEVICE_CONSTEXPR T apply(T x) const - { - return x; - } - - MIGRAPHX_DEVICE_CONSTEXPR auto pad() const - { - const auto& self = static_cast(*this); - return self.init(); - } - - template - MIGRAPHX_DEVICE_CONSTEXPR T final(T x, U) const - { - return x; - } -}; - -struct max_pool : pool_op -{ - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return lowest{}; } - - MIGRAPHX_DEVICE_CONSTEXPR auto reduce() const { return op::max{}; } -}; - -struct average_pool : pool_op -{ - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return make_tuple(0.0, 0); } - - template - MIGRAPHX_DEVICE_CONSTEXPR tuple apply(T x) const - { - return {x, 1}; - } - - MIGRAPHX_DEVICE_CONSTEXPR auto reduce() const { return op::sum{}; } - - template - MIGRAPHX_DEVICE_CONSTEXPR T final(tuple t, U) const - { - T x = t[_c<0>]; - index_int y = t[_c<1>]; - return (y == 0) ? T{0.0} : T{x / y}; - } -}; - -struct average_include_pad_pool : pool_op -{ - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return 0.0; } - - MIGRAPHX_DEVICE_CONSTEXPR auto reduce() const { return op::sum{}; } - - template - MIGRAPHX_DEVICE_CONSTEXPR T final(T x, U y) const - { - if constexpr(y == 0) - return T{0.0}; - constexpr auto scale = T{1.0} / y; - return T{x * scale}; - } -}; - -struct lpnorm_pool_base -{ -}; - -template -struct lpnorm_pool : lpnorm_pool_base, pool_op> -{ - MIGRAPHX_DEVICE_CONSTEXPR auto init() const { return 0.0; } - - template - MIGRAPHX_DEVICE_CONSTEXPR T apply(T x) const - { - if constexpr(P == 0) - return 1; - else if constexpr(P == 1) - return migraphx::abs(x); - else if constexpr(P == 2) - return x * x; - else - return migraphx::pow(migraphx::abs(x), T(P)); - } - - MIGRAPHX_DEVICE_CONSTEXPR auto pad() const { return apply(init()); } - - MIGRAPHX_DEVICE_CONSTEXPR auto reduce() const { return op::sum{}; } - - template - MIGRAPHX_DEVICE_CONSTEXPR T final(T x, U) const - { - if constexpr(P == 0) - return 1; - else if constexpr(P == 1) - return x; - else if constexpr(P == 2) - return migraphx::sqrt(x); - else - return migraphx::pow(x, 1. / P); - } -}; - -template -struct window -{ - Window win = {}; - Stride stride = {}; - Padding padding = {}; - - using rank = decltype(Window{}.size()); - - constexpr auto size() const - { - return return_c([] { return Window{}.product(); }); - } - - constexpr auto has_padding() const - { - return return_c([] { return Padding{} == 0; }); - } - - template - constexpr auto apply(OutputIndex i, F f) const - { - auto win_start = generate_array(rank{}, [&](auto j) { - diff_int dim = i[j]; - MIGRAPHX_ASSERT(win[j] >= 1); - diff_int s = stride[j]; - diff_int p = padding[j]; - return (dim * s) - p; - }); - return [=](auto j) { return f(win_start + win.multi(j)); }; - } - - template - constexpr void visit(Index i, F f) const - { - repeat(size(), apply(i, f)); - } -}; - -template -constexpr window make_window(Window w, Stride s, Padding p) -{ - return {w, s, p}; -} - -template -__device__ void pooling_reduce(Output output, F f) -{ - if constexpr(GroupSize < 2) - { - Algo::template run( - [&](auto out_idx, auto r) { r.outer([&] { output[out_idx] = f(out_idx, r); }); }); - } - else - { - auto goutput = as_vec(output, output.get_shape().lens.size() - _c<1>); - Algo::template run([&](auto out_idx, auto r) { - auto i = out_idx; - i.back() *= GroupSize; - auto result = vec_generate([&](auto) { - i.back()++; - return f(i, r); - }); - r.outer([&] { goutput[out_idx] = result; }); - }); - } -} - -template -__device__ void pooling(Op op, Window w, Output output, Input input) -{ - pooling_reduce(output, [&](auto out_idx, auto r) { - auto x = r.reduce(op.reduce(), op.init(), w.apply(out_idx, [&](auto j) { - using itype = decltype(op.apply(input[j])); - - if(j < input.get_shape().lens) - { - return op.apply(input[j]); - } - else - { - return itype(op.pad()); - } - }))(reduce::make_indices(w.size())); - return op.final(x, w.size()); - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_POOLING_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pp.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pp.hpp deleted file mode 100644 index 89b38ac24..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/pp.hpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_PP_HPP -#define MIGRAPHX_GUARD_KERNELS_PP_HPP - -// NOLINTBEGIN(*-macro-to-enum) - -#define MIGRAPHX_PP_PRIMITIVE_CAT(x, y) x##y -#define MIGRAPHX_PP_CAT(x, y) MIGRAPHX_PP_PRIMITIVE_CAT(x, y) - -#define MIGRAPHX_PP_EAT(...) -#define MIGRAPHX_PP_EXPAND(...) __VA_ARGS__ -#define MIGRAPHX_PP_COMMA(...) , - -#define MIGRAPHX_PP_IIF(c) MIGRAPHX_PP_PRIMITIVE_CAT(MIGRAPHX_PP_IIF_, c) -#define MIGRAPHX_PP_IIF_0(t, ...) __VA_ARGS__ -#define MIGRAPHX_PP_IIF_1(t, ...) t - -#define MIGRAPHX_PP_COMPL(b) MIGRAPHX_PP_PRIMITIVE_CAT(MIGRAPHX_PP_COMPL_, b) -#define MIGRAPHX_PP_COMPL_0 1 -#define MIGRAPHX_PP_COMPL_1 0 - -#define MIGRAPHX_PP_BITAND(x) MIGRAPHX_PP_PRIMITIVE_CAT(MIGRAPHX_PP_BITAND_, x) -#define MIGRAPHX_PP_BITAND_0(y) 0 -#define MIGRAPHX_PP_BITAND_1(y) y - -#define MIGRAPHX_PP_CHECK(...) MIGRAPHX_PP_CHECK_N(__VA_ARGS__, 0, ) -#define MIGRAPHX_PP_CHECK_N(x, n, ...) n -#define MIGRAPHX_PP_PROBE(x) x, 1, - -#define MIGRAPHX_PP_IS_PAREN(x) MIGRAPHX_PP_CHECK(MIGRAPHX_PP_IS_PAREN_PROBE x) -#define MIGRAPHX_PP_IS_PAREN_PROBE(...) MIGRAPHX_PP_PROBE(~) - -#define MIGRAPHX_PP_PRIMITIVE_IS_EMPTY(x) \ - MIGRAPHX_PP_CHECK(MIGRAPHX_PP_PRIMITIVE_IS_EMPTY_PROBE x()) -#define MIGRAPHX_PP_PRIMITIVE_IS_EMPTY_PROBE(...) MIGRAPHX_PP_PROBE(~) - -#define MIGRAPHX_PP_IS_EMPTY_ARG(x) \ - MIGRAPHX_PP_BITAND(MIGRAPHX_PP_COMPL(MIGRAPHX_PP_IS_PAREN(x))) \ - (MIGRAPHX_PP_PRIMITIVE_IS_EMPTY(x)) - -#define MIGRAPHX_PP_REPEAT0(m, ...) m(0, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT1(m, ...) MIGRAPHX_PP_REPEAT0(m, __VA_ARGS__) m(1, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT2(m, ...) MIGRAPHX_PP_REPEAT1(m, __VA_ARGS__) m(2, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT3(m, ...) MIGRAPHX_PP_REPEAT2(m, __VA_ARGS__) m(3, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT4(m, ...) MIGRAPHX_PP_REPEAT3(m, __VA_ARGS__) m(4, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT5(m, ...) MIGRAPHX_PP_REPEAT4(m, __VA_ARGS__) m(5, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT6(m, ...) MIGRAPHX_PP_REPEAT5(m, __VA_ARGS__) m(6, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT7(m, ...) MIGRAPHX_PP_REPEAT6(m, __VA_ARGS__) m(7, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT8(m, ...) MIGRAPHX_PP_REPEAT7(m, __VA_ARGS__) m(8, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT9(m, ...) MIGRAPHX_PP_REPEAT8(m, __VA_ARGS__) m(9, __VA_ARGS__) -#define MIGRAPHX_PP_REPEAT10(m, ...) MIGRAPHX_PP_REPEAT9(m, __VA_ARGS__) m(10, __VA_ARGS__) - -#define MIGRAPHX_PP_REPEAT(n, m, ...) \ - MIGRAPHX_PP_PRIMITIVE_CAT(MIGRAPHX_PP_REPEAT, n)(m, __VA_ARGS__) - -#define MIGRAPHX_PP_RES_ARGS() , , , , , , , , , , , , , , , - -#define MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARGS(...) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARGS_IMPL(__VA_ARGS__) - -#define MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARGS_IMPL( \ - m, delim, x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, ...) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x0) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x1) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x1) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x2) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x2) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x3) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x3) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x4) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x4) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x5) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x5) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x6) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x6) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x7) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x7) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x8) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x8) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x9) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x9) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x10) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x10) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x11) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x11) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x12) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x12) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x13) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x13) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x14) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x14) \ - MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(delim, x15) MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x15) - -#define MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARG(m, x) \ - MIGRAPHX_PP_IIF(MIGRAPHX_PP_IS_EMPTY_ARG(x))(MIGRAPHX_PP_EAT, m)(x) - -#define MIGRAPHX_PP_EACH_ARGS(m, ...) \ - MIGRAPHX_PP_EXPAND(MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARGS( \ - m, MIGRAPHX_PP_EAT, __VA_ARGS__, MIGRAPHX_PP_RES_ARGS())) - -#define MIGRAPHX_PP_TRANSFORM_ARGS(m, ...) \ - MIGRAPHX_PP_EXPAND(MIGRAPHX_PP_PRIMITIVE_TRANSFORM_ARGS( \ - m, MIGRAPHX_PP_COMMA, __VA_ARGS__, MIGRAPHX_PP_RES_ARGS())) - -// NOLINTEND(*-macro-to-enum) - -#endif // MIGRAPHX_GUARD_KERNELS_PP_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/preload.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/preload.hpp deleted file mode 100644 index 3978d0af3..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/preload.hpp +++ /dev/null @@ -1,198 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_PRELOAD_HPP -#define MIGRAPHX_GUARD_KERNELS_PRELOAD_HPP - -#include -#include -#include -#include - -namespace migraphx { - -template -struct remove_vec_impl -{ - using type = T; -}; - -template -struct remove_vec_impl> -{ - using type = T; -}; - -template -using remove_vec = typename remove_vec_impl::type; - -template -constexpr auto traverse_preload(Shapes... ss) -{ - return [=](auto f, auto... g) { - index_int offset = 0; - auto each = [&](auto x) { - using type = remove_vec; - constexpr auto s = decltype(x.get_shape()){}; - constexpr auto size = s.element_space(); - if constexpr(not s.broadcasted() or (s.elements() - size) < 64 or - not is_same{}) - return f(x, offset, false_type{}); - else - { - auto pre_offset = offset; - offset += size; - offset += offset % 4; - return f(x, pre_offset, true_type{}); - } - }; - return by(each, g...)(ss...); - }; -} - -template -constexpr index_int compute_preload_size_c(Shapes...) -{ - index_int size = 0; - traverse_preload(Shapes{}...)( - [&](auto s, auto offset, auto) { size = offset + s.element_space(); }); - return size; -} - -template -constexpr auto compute_preload_size(Shapes...) -{ - return _c(Shapes{}...)>; -} - -template -__device__ auto preload_copy(index idx, F f, __shared__ T* buffer, Ts... xs) -{ - auto invoke = [&](auto... ys) { - __syncthreads(); - f(ys...); - }; - traverse_preload(xs...)( - [&](auto x, auto offset, auto copy) { - if constexpr(copy) - { - if constexpr(decltype(tensor_vec_size(x)){} == 0) - { - auto v = auto_vectorize(x); - auto b = as_vec(tensor_vec_size(v), buffer + offset); - idx.local_stride(v.get_shape().element_space(), - [&](auto i) { b[i] = v.data()[i]; }); - return x.with(buffer + offset); - } - else - { - auto b = as_vec(tensor_vec_size(x), buffer + offset); - idx.local_stride(x.get_shape().element_space(), - [&](auto i) { b[i] = x.data()[i]; }); - return x.with(b); - } - } - else - { - return x; - } - }, - invoke); -} - -template -struct shape_type : Shape -{ - using type = T; -}; - -template -constexpr auto make_shape_type(T) -{ - return shape_type{}; -} - -template -__device__ auto preload(index idx, Ts... xs) -{ - using type = remove_vec; - constexpr auto size = decltype(compute_preload_size(make_shape_type(xs)...)){}; - const index_int max_size = 512 * sizeof(type); - return [=](auto f) { - if constexpr(size > 0 and size < max_size) - { - __shared__ type buffer[size]; - preload_copy(idx, f, buffer, xs...); - } - else - { - f(xs...); - } - }; -} - -inline __device__ auto auto_preload(index idx) -{ - return make_transform([=](auto f, auto out, auto... xs) { - preload(idx, xs...)([&](auto... ys) { f(out, ys...); }); - }); -} - -template -__device__ auto preload_copy(index idx, T x) -{ - return [=](auto f) { - if constexpr(B) - { - using type = typename T::type; - constexpr auto size = get_shape_c{}.element_space(); - __shared__ type buffer[size]; - // TODO: Always vecotrize when size > 4, and then use a second loop for remainder - constexpr auto n = find_vectorize_size([&](auto i) { return (size % i) == 0; }); - auto input = as_vec(remove_bool(x.data())); - auto b = as_vec(remove_bool(buffer)); - idx.local_stride(size / n, [&](auto i) { b[i] = input[i]; }); - return f(x.with(buffer)); - } - else - { - return f(x); - } - }; -} - -template -__device__ auto auto_preload(index idx) -{ - return make_transform([=](auto f, auto... xs) { - auto invoke = [=](auto... ys) { - if constexpr((Bs or ...)) - __syncthreads(); - f(ys...); - }; - join(invoke, preload_copy(idx, xs)...); - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_PRELOAD_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/print.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/print.hpp deleted file mode 100644 index a12424535..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/print.hpp +++ /dev/null @@ -1,270 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_PRINT_HPP -#define MIGRAPHX_GUARD_KERNELS_PRINT_HPP - -#include -#include -#include -#include - -namespace migraphx { - -template -struct on_exit -{ - F f; - G g; - template - __host__ __device__ auto operator()(T x) const - { - return f(x); - } - - __host__ __device__ ~on_exit() { f(g); } -}; - -template -constexpr auto print_type_name_probe() -{ - constexpr auto name = __PRETTY_FUNCTION__; - constexpr auto size = sizeof(__PRETTY_FUNCTION__); - constexpr auto parameter_name = "PrivateMIGraphXTypeNameProbe = "; - constexpr auto parameter_name_size = sizeof("PrivateMIGraphXTypeNameProbe = ") - 1; - constexpr auto begin = - search(name, name + size, parameter_name, parameter_name + parameter_name_size); - static_assert(begin < name + size, "Type probe not found."); - constexpr auto start = begin + parameter_name_size; - constexpr auto last = find_if(start, name + size, [](auto c) { return c == ']' or c == ';'; }); - return [=](const auto& s) { s.print_string(start, last - start); }; -} - -template -struct type_printer -{ - template - friend constexpr const Stream& operator<<(const Stream& s, type_printer) - { - print_type_name_probe()(s); - return s; - } -}; - -template -constexpr type_printer type_of() -{ - return {}; -} - -template -constexpr type_printer type_of(T) -{ - return {}; -} - -template -constexpr type_printer sub_type_of() -{ - return {}; -} - -template -constexpr type_printer sub_type_of(T) -{ - return {}; -} - -template -struct basic_printer -{ - F f; - __host__ __device__ const basic_printer& print_long(long value) const - { - f([&] { printf("%li", value); }); - return *this; - } - __host__ __device__ const basic_printer& print_ulong(unsigned long value) const - { - f([&] { printf("%lu", value); }); - return *this; - } - __host__ __device__ const basic_printer& print_char(char value) const - { - f([&] { printf("%c", value); }); - return *this; - } - __host__ __device__ const basic_printer& print_string(const char* value) const - { - f([&] { printf("%s", value); }); - return *this; - } - __host__ __device__ const basic_printer& print_string(const char* value, int size) const - { - f([&] { printf("%.*s", size, value); }); - return *this; - } - __host__ __device__ const basic_printer& print_double(double value) const - { - f([&] { printf("%f", value); }); - return *this; - } - __host__ __device__ const basic_printer& print_bool(bool value) const - { - f([&] { - if(value) - printf("true"); - else - printf("false"); - }); - return *this; - } - __host__ __device__ const basic_printer& operator<<(short value) const - { - return print_long(value); - } - __host__ __device__ const basic_printer& operator<<(unsigned short value) const - { - return print_ulong(value); - } - __host__ __device__ const basic_printer& operator<<(int value) const - { - return print_long(value); - } - __host__ __device__ const basic_printer& operator<<(unsigned int value) const - { - return print_ulong(value); - } - __host__ __device__ const basic_printer& operator<<(long value) const - { - return print_long(value); - } - __host__ __device__ const basic_printer& operator<<(unsigned long value) const - { - return print_ulong(value); - } - __host__ __device__ const basic_printer& operator<<(migraphx::half value) const - { - return print_double(value); - } - __host__ __device__ const basic_printer& operator<<(float value) const - { - return print_double(value); - } - __host__ __device__ const basic_printer& operator<<(double value) const - { - return print_double(value); - } - __host__ __device__ const basic_printer& operator<<(bool value) const - { - return print_bool(value); - } - __host__ __device__ const basic_printer& operator<<(char value) const - { - return print_char(value); - } - __host__ __device__ const basic_printer& operator<<(unsigned char value) const - { - return print_char(value); - } - __host__ __device__ const basic_printer& operator<<(const char* value) const - { - return print_string(value); - } -}; - -template -constexpr basic_printer make_printer(F f) -{ - return {f}; -} - -template -constexpr basic_printer> make_printer(F f, G g) -{ - return {{f, g}}; -} - -inline __device__ auto cout() -{ - return make_printer([](auto f) { f(); }); -} - -inline __device__ auto coutln() -{ - return make_printer([](auto f) { f(); }, [] { printf("\n"); }); -} - -template -__device__ void unsafe_print_each(Stream s, T x, Ts... xs) -{ - s << x; - each_args([&](auto xx) { s << ' ' << xx; }, xs...); -} - -template -__device__ void print_each(Stream s, Ts... xs) -{ - auto idx = make_index(); - for(auto i = 0; i < idx.nglobal(); i++) - { - if(i == idx.global) - unsafe_print_each(s, xs...); - __syncthreads(); - } -} - -template -__device__ void print_each_once(Stream s, Ts... xs) -{ - auto idx = make_index(); - if(idx.global == 0) - unsafe_print_each(s, xs...); -} - -template -__device__ void print(Ts... xs) -{ - print_each(cout(), xs...); -} - -template -__device__ void print_once(Ts... xs) -{ - print_each_once(cout(), xs...); -} - -template -__device__ void println(Ts... xs) -{ - print_each(cout(), xs..., '\n'); -} - -template -__device__ void println_once(Ts... xs) -{ - print_each_once(cout(), xs..., '\n'); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_PRINT_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ranges.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ranges.hpp deleted file mode 100644 index af32a723b..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/ranges.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_RANGES_HPP -#define MIGRAPHX_GUARD_KERNELS_RANGES_HPP - -#include - -namespace migraphx { - -template -struct iterator_range -{ - Iterator start; - Iterator last; - - constexpr Iterator begin() const { return start; } - - constexpr Iterator end() const { return last; } -}; - -constexpr iterator_range range(diff_int start, diff_int last) -{ - return {{start, {}}, {last, {}}}; -} -constexpr iterator_range range(diff_int last) { return range(0, last); } - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_RANGES_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/rank.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/rank.hpp deleted file mode 100644 index 5765b4f3e..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/rank.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_RANK_HPP -#define MIGRAPHX_GUARD_KERNELS_RANK_HPP - -namespace migraphx { - -template -struct rank : rank -{ -}; - -template <> -struct rank<0> -{ -}; - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_RANK_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/reduce.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/reduce.hpp deleted file mode 100644 index 76150cbfc..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/reduce.hpp +++ /dev/null @@ -1,785 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_REDUCE_HPP -#define MIGRAPHX_GUARD_KERNELS_REDUCE_HPP - -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { - -#if MIGRAPHX_HAS_DPP - -template -__device__ void dpp_reduce(T& in, Op op) -{ - static_assert(SubWaveSize <= MIGRAPHX_WAVEFRONTSIZE, "Too large subwave size"); - static_assert(is_power_of_2(SubWaveSize), "SubWaveSize is not a power of 2"); - if constexpr(SubWaveSize > 1) - { - auto out = dpp_mov(in); - in = op(in, out); - } - if constexpr(SubWaveSize > 2) - { - auto out = dpp_mov(in); - in = op(in, out); - } - if constexpr(SubWaveSize > 4) - { - auto out = dpp_mov(in); - in = op(in, out); - } - if constexpr(SubWaveSize > 8) - { - auto out = dpp_mov(in); - in = op(in, out); - } -#if MIGRAPHX_WAVEFRONTSIZE == 32 - if constexpr(SubWaveSize > 16) - { - auto out = dpp_swizzle<0x1e0>(in); - in = op(in, out); - } -#else - if constexpr(SubWaveSize > 16) - { - auto out = dpp_mov(in); - in = op(in, out); - } - if constexpr(SubWaveSize > 32) - { - auto out = dpp_mov(in); - in = op(in, out); - } -#endif -} - -#if defined(MIGRAPHX_USE_CLANG_TIDY) || defined(CPPCHECK) -// NOLINTNEXTLINE -#define MIGRAPHX_DPP_REDUCE_ASM_FUN(type, op, ins) \ - template \ - __device__ inline void dpp_reduce(type& x, op f) \ - { \ - (void)f; \ - x = 1; \ - } -#else -#define MIGRAPHX_DPP_IIF64(then, ...) then -#define MIGRAPHX_DPP_IIF32(then, ...) __VA_ARGS__ -#define MIGRAPHX_DPP_IF_64(x) MIGRAPHX_PP_CAT(MIGRAPHX_DPP_IIF, x) -#define MIGRAPHX_DPP_WHEN_64(x) MIGRAPHX_DPP_IF_64(x)(MIGRAPHX_PP_EXPAND, MIGRAPHX_PP_EAT) - -#define MIGRAPHX_DPP_REDUCE_ASM0(ins) #ins " %0 %0 %0 row_shr:1\n" -#define MIGRAPHX_DPP_REDUCE_ASM1(ins) #ins " %0 %0 %0 row_shr:2\n" -#define MIGRAPHX_DPP_REDUCE_ASM2(ins) #ins " %0 %0 %0 row_shr:4 bank_mask:0xe\n" -#define MIGRAPHX_DPP_REDUCE_ASM3(ins) #ins " %0 %0 %0 row_shr:8 bank_mask:0xc\n" -#define MIGRAPHX_DPP_REDUCE_ASM4(ins) #ins " %0 %0 %0 row_bcast:15 row_mask:0xa\n" -#define MIGRAPHX_DPP_REDUCE_ASM5(ins) #ins " %0 %0 %0 row_bcast:31 row_mask:0xc\n" - -#define MIGRAPHX_DPP_REDUCE_ASM_REPEAT(i, ins) \ - MIGRAPHX_PP_CAT(MIGRAPHX_DPP_REDUCE_ASM, i)(ins) "s_nop 1\n" -#define MIGRAPHX_DPP_REDUCE_ASM(n, x, ins, ...) \ - { \ - __asm__ volatile("s_nop 4\n" MIGRAPHX_PP_REPEAT(n, MIGRAPHX_DPP_REDUCE_ASM_REPEAT, ins) \ - : "=v"(x) \ - : "0"(x)); \ - __VA_ARGS__ \ - } - -#if MIGRAPHX_WAVEFRONTSIZE == 64 -#define MIGRAPHX_DPP_REDUCE_SWIZZLE(x, f) (void)f; -#else -#define MIGRAPHX_DPP_REDUCE_SWIZZLE(x, f) \ - auto y = dpp_swizzle<0x1e0>(x); \ - x = f(x, y); -#endif - -#define MIGRAPHX_DPP_REDUCE_ASM_FUN(type, op, ins) \ - template \ - __device__ inline void dpp_reduce(type& x, op f) \ - { \ - if constexpr(SubWaveSize == 2) \ - MIGRAPHX_DPP_REDUCE_ASM(0, x, ins, ); \ - if constexpr(SubWaveSize == 4) \ - MIGRAPHX_DPP_REDUCE_ASM(1, x, ins, ); \ - if constexpr(SubWaveSize == 8) \ - MIGRAPHX_DPP_REDUCE_ASM(2, x, ins, ); \ - if constexpr(SubWaveSize == 16) \ - MIGRAPHX_DPP_REDUCE_ASM(3, x, ins, ); \ - if constexpr(SubWaveSize == 32) \ - MIGRAPHX_DPP_REDUCE_ASM(MIGRAPHX_DPP_IF_64(MIGRAPHX_WAVEFRONTSIZE)(4, 3), \ - x, \ - ins, \ - MIGRAPHX_DPP_REDUCE_SWIZZLE(x, f)); \ - MIGRAPHX_DPP_WHEN_64(MIGRAPHX_WAVEFRONTSIZE) \ - (if constexpr(SubWaveSize == 64) MIGRAPHX_DPP_REDUCE_ASM(5, x, ins, )); \ - } -#endif - -// Navi21 doesn't support int32 dpp -#if defined(__gfx1030__) -// NOLINTNEXTLINE -#define MIGRAPHX_DPP_REDUCE(op, prefix, sign) \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(double, op, prefix##_f64); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(float, op, prefix##_f32); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(half, op, prefix##_f16); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(uint32_t, op, prefix##_u32); -#else -// NOLINTNEXTLINE -#define MIGRAPHX_DPP_REDUCE(op, prefix, sign) \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(double, op, prefix##_f64); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(float, op, prefix##_f32); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(half, op, prefix##_f16); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(int32_t, op, prefix##sign##32); \ - MIGRAPHX_DPP_REDUCE_ASM_FUN(uint32_t, op, prefix##_u32); -#endif - -// Note: when max and min are in int32_t, signed version of instruction needs to be used. -MIGRAPHX_DPP_REDUCE(op::sum, v_add, _u) -MIGRAPHX_DPP_REDUCE(op::product, v_mul, _u) -MIGRAPHX_DPP_REDUCE(op::max, v_max, _i) -MIGRAPHX_DPP_REDUCE(op::min, v_min, _i) - -template -__device__ void dpp_reduce(T& in, Op op) -{ - dpp_reduce(in, op); -} - -template -__device__ auto subwave_reduce(index idx, Op op, T init, Index n, F f) -{ - MIGRAPHX_ASSERT(idx.max_nlocal() == idx.nlocal() or (idx.nlocal() % SubWaveSize) == 0); - using type = decltype(index::invoke_loop(f, 0, _c<0>)); - auto x = type(init); - idx.local_subwave_stride( - n, [&](auto i, auto d) { x = op(x, index::invoke_loop(f, i, d)); }); - dpp_reduce(x, op); - return readlane(x); -} - -template -__device__ auto wave_reduce(index idx, Op op, T init, Index n, F f) -{ - return subwave_reduce(idx, op, init, n, f); -} - -template -__device__ auto block_reduce(index idx, Op op, T init, Index n, F f) -{ - MIGRAPHX_ASSERT(idx.max_nlocal() == idx.nlocal()); -#ifdef MIGRAPHX_HAS_CONST_LOCAL - if constexpr(decltype(idx.nlocal()){} == MIGRAPHX_WAVEFRONTSIZE) - return wave_reduce(idx, op, init, n, f); -#endif - constexpr index_int lanes_per_thread = MIGRAPHX_WAVEFRONTSIZE; - using type = decltype(index::invoke_loop(f, 0, _c<0>)); - __shared__ type buffer[idx.max_nlocal() / lanes_per_thread]; - auto x = type(init); - idx.local_stride(n, [&](auto i, auto d) { x = op(x, index::invoke_loop(f, i, d)); }); - dpp_reduce(x, op); - - const auto ldsidx = idx.local / lanes_per_thread; - if((idx.local % lanes_per_thread) == lanes_per_thread - 1) - { - buffer[ldsidx] = x; - } - __syncthreads(); - - type y = type(init); - for(index_int i = 0; i < idx.nlocal() / lanes_per_thread; i++) - { - y = op(y, buffer[i]); - } - return y; -} -#else -template -__device__ auto block_reduce(index idx, Op op, T init, Index n, F f) -{ - MIGRAPHX_ASSERT(idx.max_nlocal() == idx.nlocal()); - using type = decltype(index::invoke_loop(f, 0, _c<0>)); - __shared__ type buffer[idx.max_nlocal()]; - auto x = type(init); - idx.local_stride(n, [&](auto i, auto d) { x = op(x, index::invoke_loop(f, i, d)); }); - buffer[idx.local] = x; - __syncthreads(); - - for(index_int s = 1; s < idx.nlocal(); s *= 2) - { - const index_int index = 2 * s * idx.local; - if(index + s < idx.nlocal()) - { - buffer[index] = op(buffer[index], buffer[index + s]); - } - __syncthreads(); - } - return buffer[0]; -} -#endif - -template -constexpr auto reduce_slice(Input input, T i) -{ - constexpr auto lens = transform(get_shape_c{}.lens, - get_shape_c{}.lens, - [](index_int x, index_int y) -> index_int { - if(x == y) - return 1; - return x; - }); - ; - constexpr auto s = make_shape(lens, get_shape_c{}.strides); - MIGRAPHX_ASSERT((input.get_shape().index(i) + s.element_space()) <= - input.get_shape().element_space()); - return make_tensor_view(&input[i], s); -} - -namespace reduce { - -struct inner_storage_tag -{ -}; - -template -using is_inner_storage = is_base_of>>; - -template -struct lazy_inner_storage : inner_storage_tag -{ - using type = remove_reference_t()(0, _c<0>))>; - F f; - constexpr Size rsize() const { return {}; } - template - constexpr auto operator()(U j, V d) const - { - return f(j, d); - } -}; - -template -constexpr lazy_inner_storage make_lazy_inner_storage(Size, F f) -{ - return {{}, f}; -} - -template -constexpr auto make_indices(Size size) -{ - return make_lazy_inner_storage(size, [](auto j, auto) { return j; }); -} - -template -struct storage_access : F -{ - using type = R; -}; - -template -constexpr storage_access make_storage_access(F f) -{ - return {{f}}; -} - -template -constexpr auto sliced(Slicer slicer, F f) -{ - return [=](auto x, auto... xs) { - // TODO: assert all elements are the same - return f(slicer(x), slicer(xs)...); - }; -} - -template -constexpr auto compute_reduce_axis() -{ - constexpr auto lens = - transform_i(get_shape_c{}.lens, [](index_int x, index_int i) -> index_int { - if(i == Axis) - return 1; - return x; - }); - return make_shape(lens, get_shape_c{}.strides); -} - -template -constexpr auto final_reduce(T x, F f) -{ - return vec_reduce(x, f); -} - -template -constexpr auto final_reduce(array a, F f) -{ - return a.apply([&](auto x) { return final_reduce(x, f); }); -} - -template -using with_axis = decltype(compute_reduce_axis()); - -template -struct reducer_base -{ - template - __device__ decltype(auto) make_inner_slice(T&& x) const - { - if constexpr(is_inner_storage{}) - { - return x; - } - else - { - auto&& derived = static_cast(*this); - auto t = derived.slice(x); - return make_storage_access( - [=](auto i, auto...) -> auto& { return t[i]; }); - } - } - - template - constexpr auto get_size(T&& x, [[maybe_unused]] Ts&&... xs) const - { - MIGRAPHX_ASSERT(get_size(x) == get_size(xs...)); - return get_size(x); - } - - template - constexpr auto get_size(T&& x) const - { - if constexpr(is_inner_storage{}) - { - return x.rsize(); - } - else - { - auto&& derived = static_cast(*this); - auto t = derived.slice(x); - return t.size(); - } - } - - template - __device__ auto inner_sliced(F f) const - { - return [=](auto&&... xs) { return f(get_size(xs...), make_inner_slice(xs)...); }; - } - - template - static __device__ typename T::type& decl_inner_storage(const T&); - - template - __device__ auto inner(F f) const - { - return this->inner_sliced([=](auto n, auto&&... xs) { - using result_type = decltype(f(decl_inner_storage(xs)...)); - auto&& derived = static_cast(*this); - if constexpr(is_void{}) - { - derived.inner_void_impl(f, n, xs...); - } - else - { - return derived.template inner_impl(f, n, xs...); - } - }); - } - - template - __device__ auto lazy_inner(F f) const - { - return this->inner_sliced([=](auto n, auto&&... xs) { - return make_lazy_inner_storage(n, [=](auto j, auto d) { return f(xs(j, d)...); }); - }); - } - - template - __device__ auto reduce(Op op, T init, Read read) const - { - return this->inner_sliced([=](auto n, auto&&... xs) { - auto&& derived = static_cast(*this); - return derived.reduce_impl(op, init, read, n, xs...); - }); - } - - template - __device__ auto reduce(Op op, T init) const - { - return this->reduce(op, init, op::id{}); - } - - template - __device__ void outer(F f) const - { - f(); - } - - template - constexpr auto elements() const - { - auto&& derived = static_cast(*this); - using reduce_type = decltype(derived.slice(Input{})); - using value_type = typename Input::type; - constexpr auto relements = get_shape_c{}.elements(); - if constexpr(vec_size() > 1) - return relements * vec_size(); - else - return relements; - } -}; - -struct block -{ - template - struct reducer : reducer_base> - { - index idx; - Slicer slice; - - template - struct inner_storage : inner_storage_tag - { - using type = T; - array arr; - constexpr Size rsize() const { return {}; } - template - constexpr auto& operator()(U, V d) const - { - return arr[d]; - } - template - constexpr auto& operator()(U, V d) - { - return arr[d]; - } - }; - - template - __device__ auto reduce_impl(Op op, T init, Read read, N n, Ts&&... xs) const - { - return block_reduce(idx, op, init, n, [&](auto j, auto d) { - return final_reduce(read(xs(j, d)...), op); - }); - } - - template - __device__ void outer(F f) const - { - if(idx.local == 0) - f(); - } - - template - __device__ void inner_void_impl(F f, N n, Ts&&... xs) const - { - idx.local_stride(n, [&](auto j, auto d) { f(xs(j, d)...); }); - } - - template - __device__ auto inner_impl(F f, N n, Ts&&... xs) const - { - using max_iterations = decltype(idx.max_local_stride_iterations(n)); - inner_storage storage; - idx.local_stride(n, [&](auto j, auto d) { storage(j, d) = R{f(xs(j, d)...)}; }); - return storage; - } - }; - - template - static __device__ auto make(index idx, Slicer slicer) - { - return reducer{{}, idx, slicer}; - } - - template - static __device__ void run(F f) - { - auto idx = make_index(); - constexpr auto nelements = get_shape_c{}.elements(); - idx.global_stride(nelements * idx.nlocal(), [&](auto i) { - const auto out_idx = get_shape_c{}.multi(i / idx.nlocal()); - f(out_idx, make(idx, [&](auto input) { return reduce_slice(input, out_idx); })); - }); - } -}; - -struct block_large -{ - template - struct reducer : reducer_base> - { - index idx; - Slicer slice; - - template - __device__ auto reduce_impl(Op op, T init, Read read, N n, Ts&&... xs) const - { - return block_reduce(idx, op, init, index_int{n}, [&](auto j, auto d) { - return final_reduce(read(xs(j, d)...), op); - }); - } - - template - __device__ void outer(F f) const - { - if(idx.local == 0) - f(); - } - - template - __device__ void inner_void_impl(F f, N n, Ts&&... xs) const - { - idx.local_stride(index_int{n}, [&](auto j, auto d) { f(xs(j, d)...); }); - } - - template - __device__ auto inner_impl(F f, N n, Ts&&... xs) const - { - return make_lazy_inner_storage(n, [=](auto j, auto d) { return f(xs(j, d)...); }); - } - }; - - template - static __device__ auto make(index idx, Slicer slicer) - { - return reducer{{}, idx, slicer}; - } - - template - static __device__ void run(F f) - { - auto idx = make_index(); - constexpr auto nelements = get_shape_c{}.elements(); - idx.global_stride(nelements * idx.nlocal(), [&](auto i) { - const auto out_idx = get_shape_c{}.multi(i / idx.nlocal()); - f(out_idx, make(idx, [&](auto input) { return reduce_slice(input, out_idx); })); - }); - } -}; - -template -struct subwave -{ - template - struct reducer : reducer_base> - { - index idx; - Slicer slice; - - template - struct inner_storage : inner_storage_tag - { - using type = T; - array arr; - constexpr Size rsize() const { return {}; } - template - constexpr auto& operator()(U, V d) const - { - return arr[d]; - } - template - constexpr auto& operator()(U, V d) - { - return arr[d]; - } - }; - - template - __device__ auto reduce_impl(Op op, T init, Read read, N n, Ts&&... xs) const - { - return subwave_reduce(idx, op, init, n, [&](auto j, auto d) { - return final_reduce(read(xs(j, d)...), op); - }); - } - - template - __device__ void outer(F f) const - { - if(idx.local_subwave() == 0) - f(); - } - - template - __device__ void inner_void_impl(F f, N n, Ts&&... xs) const - { - idx.local_subwave_stride(n, [&](auto j, auto d) { f(xs(j, d)...); }); - } - - template - __device__ auto inner_impl(F f, N n, Ts&&... xs) const - { - using max_iterations = - decltype(idx.max_local_subwave_stride_iterations(n)); - inner_storage storage; - idx.local_subwave_stride( - n, [&](auto j, auto d) { storage(j, d) = f(xs(j, d)...); }); - return storage; - } - }; - - template - static __device__ auto make(index idx, Slicer slicer) - { - return reducer{{}, idx, slicer}; - } - - template - static __device__ void run(F f) - { - auto idx = make_index(); - constexpr auto nelements = get_shape_c{}.elements(); - idx.global_stride(nelements * idx.nlocal_subwave(), [&](auto i) { - const auto out_idx = get_shape_c{}.multi(i / idx.nlocal_subwave()); - f(out_idx, make(idx, [&](auto input) { return reduce_slice(input, out_idx); })); - }); - } -}; - -using wave = subwave; - -struct lane -{ - template - struct reducer : reducer_base> - { - index idx; - Slicer slice; - - template - __device__ auto reduce_impl(Op op, T init, Read read, N n, U&& x, Us&&... xs) const - { - using type = remove_reference_t), xs(0, _c<0>)...))>; - type r = type(init); - for(index_int j = 0; j < n; j++) - { - r = op(r, read(x(j, _c<0>), xs(j, _c<0>)...)); - } - return r; - } - - template - __device__ void outer(F f) const - { - f(); - } - - template - __device__ void inner_void_impl(F f, N n, Ts&&... xs) const - { - for(index_int j = 0; j < n; j++) - { - f(xs(j, _c<0>)...); - } - } - - template - __device__ auto inner_impl(F f, N n, Ts&&... xs) const - { - return make_lazy_inner_storage(n, [=](auto j, auto d) { return f(xs(j, d)...); }); - } - }; - template - static __device__ auto make(index idx, Slicer slicer) - { - return reducer{{}, idx, slicer}; - } - - template - static __device__ void run(F f) - { - auto idx = make_index(); - constexpr auto nelements = get_shape_c{}.elements(); - idx.global_stride(nelements, [&](auto i) { - const auto out_idx = get_shape_c{}.multi(i); - f(out_idx, make(idx, [&](auto input) { return reduce_slice(input, out_idx); })); - }); - } -}; - -// TODO: Remove these in the future when they can be selected in the compiler class -template -constexpr auto pick_block() -{ - using nlocal = decltype(index{}.max_nlocal()); - if constexpr(RElements < nlocal{} * 256) - return block{}; - else - return block_large{}; -} -template -using auto_block = decltype(pick_block()); - -template -constexpr auto reduce_elements_with_axis() -{ - constexpr auto s = get_shape_c{}; - return s.lens[Axis]; -} - -} // namespace reduce - -template -__device__ void -simple_reduce(Op op, T init, Input input, Output output, ReadInput read, WriteOuput write) -{ - Algo::template run([&](auto out_idx, auto r) { - auto x = r.reduce(op, init, read)(input); - r.outer([&] { output[out_idx] = write(x); }); - }); -} - -template -__device__ void fused_reduce(Output output_pack, Assign assign, F f) -{ - Algo::template run([&](auto out_idx, auto r) { - auto result_tuple = f(r, out_idx); - unpack_each( - [&](auto output, auto result) { - if constexpr(reduce::is_inner_storage{}) - { - r.inner([&](auto& y, auto x) { assign(y, x); })(output, result); - } - else - { - r.outer([&] { assign(output[out_idx], implicit_conversion(result)); }); - } - }, - output_pack, - result_tuple); - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_REDUCE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/roialign.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/roialign.hpp deleted file mode 100644 index b7d7216c6..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/roialign.hpp +++ /dev/null @@ -1,229 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_ROIALIGN_HPP -#define MIGRAPHX_GUARD_KERNELS_ROIALIGN_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { - -struct max_pool -{ - MIGRAPHX_DEVICE_CONSTEXPR auto init() { return lowest{}; } - - template - MIGRAPHX_DEVICE_CONSTEXPR T operator()(T x, T y) - { - return max(x, y); - } - - template - MIGRAPHX_DEVICE_CONSTEXPR T final(T x, index_int) - { - return (x); - } -}; - -struct avg_pool -{ - MIGRAPHX_DEVICE_CONSTEXPR auto init() { return 0.0; } - - template - MIGRAPHX_DEVICE_CONSTEXPR T operator()(T x, T y) - { - return x + y; - } - - template - MIGRAPHX_DEVICE_CONSTEXPR T final(T x, index_int y) - { - return (y == 0) ? T{0.0} : T{x / y}; - } -}; - -template -MIGRAPHX_DEVICE_CONSTEXPR typename Iterator::value_type bilinear_interpolate( - const Iterator data, const array& dims, array xy, Op pooling) -{ - array low{}; - array high{}; - for(index_int ii = 0; ii < xy.size(); ++ii) - { - if(xy[ii] < -1.0f or xy[ii] > dims[ii]) - { - return implicit_conversion(0); - } - - xy[ii] = migraphx::max(xy[ii], 0.0f); - low[ii] = xy[ii]; - high[ii] = low[ii] + 1; - if(low[ii] >= dims[ii] - 1) - { - xy[ii] = high[ii] = low[ii] = dims[ii] - 1; - } - } - array locs = {low[0] * dims[1] + low[1], - low[0] * dims[1] + high[1], - high[0] * dims[1] + low[1], - high[0] * dims[1] + high[1]}; - - float ly = xy[0] - low[0]; - float lx = xy[1] - low[1]; - float hy = 1.0f - ly; - float hx = 1.0f - lx; - // do calculations in floating point and convert final result to required type - array ws = {hy * hx, hy * lx, ly * hx, ly * lx}; - - auto v01 = pooling(data[locs[0]] * ws[0], data[locs[1]] * ws[1]); - auto v23 = pooling(data[locs[2]] * ws[2], data[locs[3]] * ws[3]); - return implicit_conversion(pooling(v01, v23)); -} - -template -MIGRAPHX_DEVICE_CONSTEXPR auto calc_pooling(const Iterator& data, - const array& roi_starts, - const array& bin_size, - const array& idx, - const array& bin_grid_size, - const array& dims, - float roi_offset, - Op op) -{ - using in_dtype = typename Iterator::value_type; - in_dtype output_val = in_dtype{op.init()}; - const int64_t count = bin_grid_size[0] * bin_grid_size[1]; - dfor(bin_grid_size[0], bin_grid_size[1])([&](auto iy, auto ix) { - array id = {iy, ix}; - array locs = - roi_starts + idx * bin_size + bin_size * (id + 0.5f) / bin_grid_size + roi_offset; - - auto val = bilinear_interpolate(data, dims, locs, op); - output_val = op(output_val, val); - }); - return op.final(output_val, count); -} - -template -struct roalign_settings -{ - T1 roi_offset{}; - T2 is_avg_pooling{}; - T3 sampling_ratio{}; - T4 spatial_scale{}; -}; - -template -constexpr roalign_settings make_roalign_settings(Ts... xs) -{ - return {xs...}; -} - -template -__device__ void roialign(const T& x_t, const U& rois_t, const V& ind_t, W& y_t, Settings s) -{ - auto index = make_index(); - const auto x = x_t.begin(); - const auto rois = rois_t.begin(); - const auto ind = ind_t.begin(); - // input shape - auto x_lens = x_t.get_shape().lens; - auto channel_num = x_lens[1]; - // input dims of height and width, in all 2-dim arrays, the first dim - // is for height and second dim is for width - array in_dims = {x_lens[2], x_lens[3]}; - - const auto stride = index.nglobal(); - auto out_s = y_t.get_shape(); - auto roi_column_num = rois_t.get_shape().lens[1]; - - // output dims of height and width, in all 2-dim arrays, the first dim - // is for height and second dim is for width - const auto& out_lens = out_s.lens; - array out_dims = {out_lens[2], out_lens[3]}; - - for(index_int i = index.global; i < out_s.elements(); i += stride) - { - auto idx = out_s.multi(i); - int n = idx[0]; - int c = idx[1]; - int ph = idx[2]; - int pw = idx[3]; - - const auto offset_rois = rois + (n * roi_column_num); - const int batch_ind = ind[n]; - - array roi_starts = { - static_cast(offset_rois[1]) * static_cast(s.spatial_scale), - static_cast(offset_rois[0]) * static_cast(s.spatial_scale)}; - array roi_ends = { - static_cast(offset_rois[3]) * static_cast(s.spatial_scale), - static_cast(offset_rois[2]) * static_cast(s.spatial_scale)}; - - array roi_size{}; - array bin_size{}; - array bin_grid_size{}; - - for(index_int ii = 0; ii < roi_size.size(); ++ii) - { - roi_size[ii] = roi_ends[ii] - roi_starts[ii]; - roi_size[ii] = migraphx::max(roi_size[ii], 1.0f); - - bin_size[ii] = roi_size[ii] / out_dims[ii]; - bin_grid_size[ii] = (s.sampling_ratio > 0) - ? s.sampling_ratio - : migraphx::ceil(roi_size[ii] / out_dims[ii]); - } - - const auto offset_x = x + ((batch_ind * channel_num + c) * in_dims[0] * in_dims[1]); - if constexpr(s.is_avg_pooling) - { - y_t[i] = calc_pooling(offset_x, - roi_starts, - bin_size, - {ph, pw}, - bin_grid_size, - in_dims, - s.roi_offset, - avg_pool{}); - } - else - { - y_t[i] = calc_pooling(offset_x, - roi_starts, - bin_size, - {ph, pw}, - bin_grid_size, - in_dims, - s.roi_offset, - max_pool{}); - } - } -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatter.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatter.hpp deleted file mode 100644 index efe5fe347..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatter.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_SCATTER_ELEMENTS_HPP -#define MIGRAPHX_GUARD_KERNELS_SCATTER_ELEMENTS_HPP - -#include -#include -#include - -namespace migraphx { - -// Checks and skips out of bounds indices if SkipOutOfBounds is true. -// Otherwise does not check and underfined behavior if out of bounds. -template -__device__ void scatter(const T& indices_t, const U& updates_t, const V& output_t, F f) -{ - auto gpu_index = make_index(); - auto indices_shape = indices_t.get_shape(); - auto output_shape = output_t.get_shape(); - auto axis_dim_size = output_shape.lens[Axis]; - - gpu_index.global_stride(indices_shape.elements(), [&](auto i) { - auto out_idx = indices_shape.multi(i); - auto index = indices_t[i]; - index = index < 0 ? index + axis_dim_size : index; - if constexpr(SkipOutOfBounds) - { - if(index < 0) - { - return; - } - } - out_idx[Axis] = index; - if constexpr(SkipOutOfBounds) - { - if(not equal( - out_idx.begin(), out_idx.end(), output_shape.lens.begin(), [](auto x, auto y) { - return x < y; - })) - { - return; - } - } - f(output_t[out_idx], updates_t[i]); - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatter_reduction_modes.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatter_reduction_modes.hpp deleted file mode 100644 index 166552a84..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatter_reduction_modes.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_SCATTER_REDUCTION_MODES_HPP -#define MIGRAPHX_GUARD_KERNELS_SCATTER_REDUCTION_MODES_HPP - -#include -#include -#include - -namespace migraphx { - -struct assign_none -{ - template - MIGRAPHX_DEVICE_CONSTEXPR void operator()(T& x, U y) const - { - x = y; - } -}; - -struct assign_add -{ - template - MIGRAPHX_DEVICE_CONSTEXPR void operator()(T& x, U y) const - { - atomic_assign(x, y, op::sum{}); - } -}; - -struct assign_mul -{ - template - MIGRAPHX_DEVICE_CONSTEXPR void operator()(T& x, U y) const - { - atomic_assign(x, y, op::product{}); - } -}; - -struct assign_max -{ - template - MIGRAPHX_DEVICE_CONSTEXPR void operator()(T& x, U y) const - { - atomic_assign(x, y, op::max{}); - } -}; - -struct assign_min -{ - template - MIGRAPHX_DEVICE_CONSTEXPR void operator()(T& x, U y) const - { - atomic_assign(x, y, op::min{}); - } -}; - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatternd.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatternd.hpp deleted file mode 100644 index dee649e8c..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/scatternd.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_SCATTERND_HPP -#define MIGRAPHX_GUARD_KERNELS_SCATTERND_HPP - -#include -#include -#include - -namespace migraphx { - -template -__device__ void scatternd(const T& indices_t, const U& updates_t, const V& output_t, F f) -{ - auto index = make_index(); - auto updates_shape = updates_t.get_shape(); - - index.global_stride(updates_shape.elements(), [&](auto i) { - auto output_shape = output_t.get_shape(); - - auto indices_shape = indices_t.get_shape(); - auto k = indices_shape.lens.back(); - auto q = indices_shape.lens.size(); - - auto updates_idx = updates_shape.multi(i); - auto indices_idx = indices_shape.multi(0); - copy(updates_idx.begin(), updates_idx.begin() + q - 1, indices_idx.begin()); - - auto index_start = indices_t.begin() + indices_shape.index(indices_idx); - auto index_end = index_start + k; - auto out_idx = output_shape.multi(0); - copy(index_start, index_end, out_idx.begin()); - copy(updates_idx.begin() + q - 1, updates_idx.end(), out_idx.begin() + k); - - f(output_t[out_idx], updates_t[i]); - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/shape.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/shape.hpp deleted file mode 100644 index 0828328d7..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/shape.hpp +++ /dev/null @@ -1,201 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_SHAPE_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_SHAPE_HPP - -#include -#include -#include -#include - -namespace migraphx { - -template -struct shape : equality_comparable> -{ - using shape_type = shape; - using index_array = typename Lens::base_array; - Lens lens = {}; - Strides strides = {}; - - constexpr shape() = default; - - constexpr shape(Lens l, Strides s) : lens(l), strides(s) {} - - constexpr auto elements() const { return _c; } - - constexpr auto element_space() const { return _c; } - - constexpr auto packed() const { return not skips() and elements() == element_space(); } - constexpr auto broadcasted() const { return _c; } - constexpr auto transposed() const - { - return return_c([] { - auto lstrides = Strides{}; - if(shape{}.broadcasted()) - { - index_array s{}; - auto out = copy_if( - lstrides.begin(), lstrides.end(), s.begin(), [](auto x) { return x != 0; }); - return not is_sorted(s.begin(), out, greater{}); - } - else - { - return not is_sorted(lstrides.begin(), lstrides.end(), greater{}); - } - }); - } - constexpr auto skips() const - { - return return_c([] { - auto lstrides = Strides{}; - return none_of(lstrides.begin(), lstrides.end(), [](auto x) { return x == 1; }); - }); - } - - constexpr auto standard() const { return packed() and not transposed(); } - - constexpr index_int index(index_array x) const { return x.dot(strides); } - - constexpr index_int index(index_int i) const - { - if(this->standard()) - { - MIGRAPHX_ASSERT(i == compute_index(i)); - return i; - } - else - { - return compute_index(i); - } - } - - constexpr index_int compute_index(index_int i) const - { - const auto rank = this->lens.size(); - index_int s = 1; - index_int result = 0; - for(index_int j = 0; j < rank; j++) - { - const index_int k = rank - j - 1; - const index_int stride = this->strides[k]; - const index_int len = this->lens[k]; - const index_int slen = s * len; - const index_int idx = (i % slen) / s; - result += stride * idx; - s = slen; - } - return result; - } - - /// Convert single index into a multi-index - constexpr index_array multi(index_int idx) const { return lens.multi(idx); } - - /// Convert multi-index into a single index - constexpr index_int single(index_array idx) const - { - if(idx.empty()) - return 0; - return inner_product(lens.begin() + 1, lens.end(), idx.begin(), idx.back()); - } - - constexpr shape get_shape() const { return *this; } - - template - friend constexpr bool operator==(const shape& x, const shape& y) - { - return x.lens == y.lens and x.strides == y.strides; - } - - template - friend constexpr const Stream& operator<<(const Stream& ss, const shape& s) - { - ss << "{" << s.lens << "}, {" << s.strides << "}"; - return ss; - } -}; - -template -constexpr auto calculate_strides(Lens) -{ - return return_array_c([] { - Lens lens{}; - array strides{1}; - const auto n = lens.size() - 1; - index_int stride = 1; - for(index_int i = 0; i < n; i++) - { - auto ri = n - i; - stride *= lens[ri]; - strides[ri - 1] = stride; - } - return strides; - }); -} - -template -constexpr shape make_shape(Lens lens, Strides strides) -{ - return {lens, strides}; -} - -template -constexpr auto make_shape(Lens lens) -{ - return make_shape(lens, calculate_strides(lens)); -} - -template -constexpr auto reorder_shape(Shape, Permutation) -{ - constexpr auto lens = return_array_c([] { return reorder_dims(Shape{}.lens, Permutation{}); }); - constexpr auto strides = - return_array_c([] { return reorder_dims(Shape{}.strides, Permutation{}); }); - return make_shape(lens, strides); -} - -template -constexpr auto make_shape_from_permutation(Lens, Permutation) -{ - constexpr auto new_lens = reorder_dims(Lens{}, Permutation{}); - return reorder_shape(make_shape(new_lens), invert_permutation(Permutation{})); -} - -template -constexpr auto make_packed_shape(Shape) -{ - constexpr auto s = Shape{}; - if constexpr(s.packed()) - { - return s; - } - else - { - return make_shape_from_permutation(s.lens, find_permutation(s)); - } -} - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/softmax.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/softmax.hpp deleted file mode 100644 index e9c2ac36f..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/softmax.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_SOFTMAX_HPP -#define MIGRAPHX_GUARD_KERNELS_SOFTMAX_HPP - -#include -#include - -namespace migraphx { - -template -__device__ void softmax(Input input1, Output output) -{ - using block = reduce::auto_block()>; - block::template run>([&](auto, auto r) { - auto x = r.inner(op::id{})(input1); -#ifdef MIGRAPHX_USE_FAST_SOFTMAX - const auto c = vec_at(r.slice(input1)[0], 0); -#else - const auto c = r.reduce(op::max{}, lowest{}, op::id{})(x); -#endif - r.inner([&](auto& x1) { x1 = migraphx::exp(x1 - c); })(x); - auto batch_sum = - r.reduce(op::sum{}, 0, [](auto x1) { return migraphx::convert(x1); })(x); - r.inner([&](auto& y, auto x1) { y = implicit_conversion(x1 / batch_sum); })(output, x); - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_SOFTMAX_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tensor_view.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tensor_view.hpp deleted file mode 100644 index e959ed6be..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tensor_view.hpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_TENSOR_VIEW_HPP -#define MIGRAPHX_GUARD_KERNELS_TENSOR_VIEW_HPP - -#include -#include -#include -#include - -namespace migraphx { - -template -struct tensor_view_iterator_read -{ - T* view; - constexpr auto& operator()(MIGRAPHX_CAPTURE_SOURCE_LOCATION(index_int) n) const - { - MIGRAPHX_ASSERT(view != nullptr); - return (*view)[n]; - } -}; - -template -struct tensor_view -{ - using type = T; - using shape_type = Shape; - using index_array = typename Shape::index_array; - using iterator = basic_iota_iterator, index_int>; - - constexpr Shape get_shape() const { return Shape{}; } - constexpr auto size() const { return get_shape().elements(); } - - struct index_to_offset - { - index_int offset; - template - constexpr index_to_offset(U i) : offset(Shape{}.index(i)) - { - } - }; - - constexpr T& operator[](MIGRAPHX_CAPTURE_SOURCE_LOCATION(index_to_offset) i) const - { - index_to_offset ito = i; - MIGRAPHX_WARN(ito.offset < get_shape().element_space(), - i, - "Out of bounds access at offset: ", - ito.offset); - return x[ito.offset]; - } - - constexpr T* data() const { return x; } - - constexpr auto begin() const { return iterator{0, {this}}; } - constexpr auto end() const { return iterator{this->size(), {this}}; } - - constexpr auto begin_at(index_array i) const - { - MIGRAPHX_ASSERT(get_shape().single(i) < get_shape().elements()); - MIGRAPHX_ASSERT(get_shape().index(i) < get_shape().element_space()); - return iterator{get_shape().single(i), {this}}; - } - - template - constexpr tensor_view with(U* y) const - { - static_assert(sizeof(T) == sizeof(U), "Not the same size"); - return {y}; - } - - T* x; -}; - -template -using get_shape_c = typename T::shape_type; - -template -constexpr tensor_view make_tensor_view(T* x, Shape) -{ - return {x}; -} - -template -constexpr auto reorder_tensor_view(T x, Permutation perm) -{ - return make_tensor_view(x.data(), reorder_shape(x.get_shape(), perm)); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_TENSOR_VIEW_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tile.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tile.hpp deleted file mode 100644 index 1f11b214f..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tile.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_TILE_HPP -#define MIGRAPHX_GUARD_KERNELS_TILE_HPP - -#include -#include -#include -#include - -namespace migraphx { - -struct tile -{ - template - static constexpr auto pad_shape(Shape) - { - constexpr Shape s{}; - constexpr auto axis = s.strides.size() - _c<1>; - constexpr auto strides = transform_i(s.strides, [](auto stride, auto i) { - if constexpr(i == decltype(axis){}) - { - // Pad by 1 element extra to avoid memory bank conflicts - return stride + 1; - } - else - { - return stride; - } - }); - return make_shape(s.lens, strides); - } - struct load - { - template - static __device__ auto copy(index idx, T x) - { - return [=](auto f) { - using type = typename T::type; - constexpr auto s = pad_shape(make_packed_shape(get_shape_c{})); - constexpr auto size = s.element_space(); - __shared__ type buffer[size]; - auto b = make_tensor_view(buffer, s); - local_tensor_copy(idx, x, b); - f(b); - }; - } - }; - struct store - { - template - static __device__ auto copy(index idx, T x) - { - return [=](auto f) { - using type = typename T::type; - constexpr auto s = pad_shape(make_packed_shape(get_shape_c{})); - constexpr auto size = s.element_space(); - __shared__ type buffer[size]; - auto b = make_tensor_view(buffer, s); - f(b); - local_tensor_copy(idx, b, x); - }; - } - }; - struct none - { - template - static __device__ auto copy(index, T x) - { - return [=](auto f) { f(x); }; - } - }; - - template - static constexpr auto slice(T x, index_int group, InnerLens, OuterLens) - { - constexpr auto outer_strides = - transform(x.get_shape().strides, InnerLens{}, [](auto stride, auto inner_len) { - return stride * inner_len; - }); - constexpr auto is = make_shape(InnerLens{}, x.get_shape().strides); - constexpr auto os = make_shape(OuterLens{}, outer_strides); - auto offset = os.index(group); - MIGRAPHX_ASSERT((os.element_space() + is.element_space()) == - (x.get_shape().element_space() + _c<1>)); - MIGRAPHX_ASSERT((is.elements() + group) <= x.get_shape().elements()); - MIGRAPHX_ASSERT((is.element_space() + offset) <= x.get_shape().element_space()); - return make_tensor_view(x.data() + offset, is); - } - - template - static __device__ auto auto_slice(index idx) - { - return make_transform([=](auto f, auto... xs) { - idx.group_stride(OuterLens{}.product(), - [=](auto group) { f(slice(xs, group, InnerLens{}, OuterLens{})...); }); - }); - } - - template - static __device__ auto auto_copy(index idx) - { - return make_transform([=](auto f, auto... xs) { - static_assert(sizeof...(Modes) == sizeof...(xs)); - auto invoke = [=](auto... ys) { - if constexpr((is_same{} or ...)) - __syncthreads(); - f(ys...); - if constexpr((is_same{} or ...)) - __syncthreads(); - }; - join(invoke, Modes::copy(idx, xs)...); - }); - } -}; - -template -__device__ auto tile_stride(index idx) -{ - if constexpr(Tiled) - { - return [=](auto... xs) { return idx.local_stride(xs...); }; - } - else - { - return [=](auto... xs) { return idx.global_stride(xs...); }; - } -} - -template -__device__ auto auto_tile(InnerLens, OuterLens) -{ - if constexpr((is_same{} and ...)) - { - return transform_args(); - } - else - { - auto idx = make_index(); - return transform_args(tile::auto_slice(idx), - tile::auto_copy(idx)); - } -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_TILE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tuple.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tuple.hpp deleted file mode 100644 index c54a9f4d3..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/tuple.hpp +++ /dev/null @@ -1,184 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#ifndef MIGRAPHX_GUARD_KERNELS_TUPLE_HPP -#define MIGRAPHX_GUARD_KERNELS_TUPLE_HPP - -#include - -namespace migraphx { - -namespace tuple_detail { - -template -struct element_storage -{ - [[no_unique_address]] T element; -}; - -template -constexpr const auto& get_element(const element_storage& x) -{ - return x.element; -} - -template -constexpr auto& get_element(element_storage& x) -{ - return x.element; -} - -struct unpack_t -{ -}; - -template -struct tuple_storage; - -template -struct tuple_storage, Ts...> : element_storage... -{ - template - constexpr tuple_storage(Us... ys) : element_storage{static_cast(ys)}... - { - } - - template - constexpr tuple_storage(unpack_t, U y) : element_storage{static_cast(y[_c])}... - { - } - - template - constexpr auto operator()(F f) const - { - return f(static_cast&>(*this).element...); - } - - template - constexpr auto operator()(F f) - { - return f(static_cast&>(*this).element...); - } - - template - constexpr auto& operator[](IntegralConstant i) - { - static_assert(i < sizeof...(Ts), "Out of bounds tuple access"); - return get_element(*this); - } - - template - constexpr auto& operator[](IntegralConstant i) const - { - static_assert(i < sizeof...(Ts), "Out of bounds tuple access"); - return get_element(*this); - } - - constexpr index_constant size() const { return {}; } - constexpr auto empty() const { return size() == _c<0>; } -}; - -template -using tuple_base = tuple_detail::tuple_storage::type, Ts...>; - -} // namespace tuple_detail - -// NOLINTNEXTLINE -#define MIGRAPHX_DEVICE_TUPLE_OP(op, binary_op) \ - template \ - constexpr tuple& operator op(const tuple& rhs) \ - { \ - (*this)( \ - [&](auto&... xs) { rhs([&](const auto&... ys) { swallow{((xs op ys), 0)...}; }); }); \ - return *this; \ - } \ - template \ - friend constexpr auto operator binary_op(const tuple& lhs, const tuple& rhs) \ - { \ - using result = tuple() binary_op declval())...>; \ - return lhs([&](auto&... xs) { \ - return rhs([&](const auto&... ys) { return result{xs binary_op ys...}; }); \ - }); \ - } - -template -struct tuple : tuple_detail::tuple_base -{ - using base = tuple_detail::tuple_base; - - constexpr tuple() : base(Ts{}...) {} - - template {} and ...))> - constexpr tuple(Us... ys) : base(ys...) - { - } - - template {} and ...))> - constexpr tuple(tuple y) : base(tuple_detail::unpack_t{}, y) - { - } - - MIGRAPHX_DEVICE_TUPLE_OP(+=, +) - MIGRAPHX_DEVICE_TUPLE_OP(-=, -) - MIGRAPHX_DEVICE_TUPLE_OP(*=, *) - MIGRAPHX_DEVICE_TUPLE_OP(/=, /) - MIGRAPHX_DEVICE_TUPLE_OP(%=, %) - MIGRAPHX_DEVICE_TUPLE_OP(&=, &) - MIGRAPHX_DEVICE_TUPLE_OP(|=, |) - MIGRAPHX_DEVICE_TUPLE_OP(^=, ^) - - friend constexpr bool operator==(const tuple& x, const tuple& y) - { - return x([&](const auto&... xs) { - return y([&](const auto&... ys) { return ((xs == ys) and ...); }); - }); - } - friend constexpr bool operator!=(const tuple& x, const tuple& y) { return not(x == y); } - friend constexpr bool operator<(const tuple& x, const tuple& y) - { - return x([&](const auto&... xs) { - return y([&](const auto&... ys) { - return fold([&](auto a, auto b) { return a == 0 ? b() : a; })(0, [&] { - return (xs < ys) ? -1 : (ys < xs) ? 1 : 0; - }...); - }); - }) < 0; - } - friend constexpr bool operator>(const tuple& x, const tuple& y) { return y < x; } - friend constexpr bool operator<=(const tuple& x, const tuple& y) { return not(x > y); } - friend constexpr bool operator>=(const tuple& x, const tuple& y) { return not(x < y); } -}; - -template -constexpr tuple make_tuple(Ts... xs) -{ - return {xs...}; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_TUPLE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/type_traits.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/type_traits.hpp deleted file mode 100644 index 24b7d4a5b..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/type_traits.hpp +++ /dev/null @@ -1,289 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_TYPE_TRAITS_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_TYPE_TRAITS_HPP - -#include -#include - -namespace migraphx { - -template -using void_t = void; - -template -U private_declval(int); - -template -T private_declval(long); - -template -auto declval() noexcept -> decltype(private_declval(0)); - -template -struct is_callable_impl : false_type -{ -}; - -template -struct is_callable_impl()(declval()...))>, F, Ts...> : true_type -{ -}; - -template -using is_callable = is_callable_impl; - -template -struct type_identity -{ - using type = T; -}; - -template -struct enable_if -{ -}; - -template -struct enable_if -{ - using type = T; -}; - -template -using enable_if_t = typename enable_if::type; - -template -struct conditional -{ - using type = T; -}; - -template -struct conditional -{ - using type = F; -}; - -template -using conditional_t = typename conditional::type; - -// NOLINTNEXTLINE -#define MIGRAPHX_BUILTIN_TYPE_TRAIT1(name) \ - template \ - struct name : bool_constant<__##name(T)> \ - { \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_BUILTIN_TYPE_TRAIT2(name) \ - template \ - struct name : bool_constant<__##name(T, U)> \ - { \ - } - -// NOLINTNEXTLINE -#define MIGRAPHX_BUILTIN_TYPE_TRAITN(name) \ - template \ - struct name : bool_constant<__##name(Ts...)> \ - { \ - } - -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_arithmetic); -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_destructible); -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_nothrow_destructible); -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_pointer); -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_scalar); -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_signed); -// MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_void); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_abstract); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_aggregate); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_array); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_class); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_compound); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_const); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_empty); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_enum); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_final); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_floating_point); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_function); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_fundamental); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_integral); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_literal_type); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_lvalue_reference); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_member_function_pointer); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_member_object_pointer); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_member_pointer); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_object); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_pod); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_polymorphic); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_reference); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_rvalue_reference); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_standard_layout); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_trivial); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_trivially_copyable); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_trivially_destructible); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_union); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_unsigned); -MIGRAPHX_BUILTIN_TYPE_TRAIT1(is_volatile); -MIGRAPHX_BUILTIN_TYPE_TRAIT2(is_assignable); -MIGRAPHX_BUILTIN_TYPE_TRAIT2(is_base_of); -MIGRAPHX_BUILTIN_TYPE_TRAIT2(is_convertible); -MIGRAPHX_BUILTIN_TYPE_TRAIT2(is_nothrow_assignable); -MIGRAPHX_BUILTIN_TYPE_TRAIT2(is_same); -MIGRAPHX_BUILTIN_TYPE_TRAIT2(is_trivially_assignable); -MIGRAPHX_BUILTIN_TYPE_TRAITN(is_constructible); -MIGRAPHX_BUILTIN_TYPE_TRAITN(is_nothrow_constructible); -MIGRAPHX_BUILTIN_TYPE_TRAITN(is_trivially_constructible); - -template -struct remove_cv -{ - using type = T; -}; - -template -struct remove_cv : remove_cv -{ -}; - -template -struct remove_cv : remove_cv -{ -}; - -template -using remove_cv_t = typename remove_cv::type; - -template -struct remove_reference -{ - using type = T; -}; -template -struct remove_reference -{ - using type = T; -}; -template -struct remove_reference -{ - using type = T; -}; - -template -using remove_reference_t = typename remove_reference::type; - -template -struct add_pointer : type_identity::type*> -{ -}; - -template -using add_pointer_t = typename add_pointer::type; - -template -struct is_void : is_same> -{ -}; - -template -struct common_type; - -template -struct common_type -{ - using type = T; -}; - -template -struct common_type -{ - using type = decltype(true ? declval() : declval()); -}; - -template -struct common_type -{ - using type = typename common_type::type, Us...>::type; -}; - -template -using common_type_t = typename common_type::type; - -#define MIGRAPHX_REQUIRES(...) enable_if_t<__VA_ARGS__, int> = 0 - -constexpr unsigned long long int_max(unsigned long n) -{ - // Note, left shift cannot be used to get the maximum value of int64_type or - // uint64_type because it is undefined behavior to left shift 64 bits for - // these types - if(n == sizeof(int64_t)) - return -1; - return (1ull << (n * 8)) - 1; -} - -template {} or is_floating_point{} or - is_same{})> -constexpr T numeric_max() -{ - if constexpr(is_integral{}) - { - if constexpr(is_unsigned{}) - return int_max(sizeof(T)); - else - return int_max(sizeof(T)) / 2; - } - else if constexpr(is_same{}) - return __DBL_MAX__; - else if constexpr(is_same{}) - return __FLT_MAX__; - else if constexpr(is_same{}) - return __FLT16_MAX__; - else if constexpr(is_same{}) - return 338953138925153547590470800371487866880.000000; - else - return 0; -} - -template -constexpr auto numeric_lowest() -> decltype(numeric_max()) -{ - if constexpr(is_integral{}) - { - if constexpr(is_unsigned{}) - return 0; - else - return -numeric_max() - 1; - } - else - { - return -numeric_max(); - } -} - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/types.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/types.hpp deleted file mode 100644 index c88343ce1..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/types.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_TYPES_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_KERNELS_TYPES_HPP - -#include - -namespace migraphx { - -#if defined(MIGRAPHX_USE_HIPRTC) -using int8_t = signed char; -using uint8_t = unsigned char; -using int16_t = signed short; -using uint16_t = unsigned short; -using int32_t = signed int; -using uint32_t = unsigned int; -using int64_t = signed long long; -using uint64_t = unsigned long long; -#elif defined(MIGRAPHX_USE_HIPRTC) -using int8_t = __hip_int8_t; -using uint8_t = __hip_uint8_t; -using int16_t = __hip_int16_t; -using uint16_t = __hip_uint16_t; -using int32_t = __hip_int32_t; -using uint32_t = __hip_uint32_t; -using int64_t = __hip_int64_t; -using uint64_t = __hip_uint64_t; -#else -using int8_t = std::int8_t; -using uint8_t = std::uint8_t; -using int16_t = std::int16_t; -using uint16_t = std::uint16_t; -using int32_t = std::int32_t; -using uint32_t = std::uint32_t; -using int64_t = std::int64_t; -using uint64_t = std::uint64_t; -#endif // MIGRAPHX_USE_HIPRTC -using index_int = uint32_t; -using diff_int = int32_t; -using uintptr_t = uint64_t; - -static_assert(sizeof(int8_t) == 1, "int8_t must be 1 bytes"); -static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 bytes"); -static_assert(sizeof(int16_t) == 2, "int16_t must be 2 bytes"); -static_assert(sizeof(uint16_t) == 2, "uint16_t must be 2 bytes"); -static_assert(sizeof(int32_t) == 4, "int32_t must be 4 bytes"); -static_assert(sizeof(uint32_t) == 4, "uint32_t must be 4 bytes"); -static_assert(sizeof(int64_t) == 8, "int64_t must be 8 bytes"); -static_assert(sizeof(uint64_t) == 8, "uint64_t must be 8 bytes"); - -#define MIGRAPHX_DEVICE_CONSTEXPR constexpr __device__ __host__ // NOLINT - -template -using vec = T __attribute__((ext_vector_type(N))); - -using half = _Float16; -using half2 = migraphx::vec; -using bf16 = __bf16; - -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/unpack_int4.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/unpack_int4.hpp deleted file mode 100644 index 35ffcff7a..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/unpack_int4.hpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_UNPACK_INT4_HPP -#define MIGRAPHX_GUARD_KERNELS_UNPACK_INT4_HPP - -#include "migraphx/kernels/types.hpp" -#include -#include - -namespace migraphx { - -template -__device__ void unpack_int4(Output output, Input input) -{ - const auto input_shape = input.get_shape(); - - make_index().global_stride(input_shape.elements(), [&](auto i) { - auto idx = input_shape.multi(i); - idx[Axis] *= 2; - const auto input_val = input[i]; - - // unpack_int4 op's normalize_compute_shape will ensure that Input::type is either uint8_t - // or int8_t - if constexpr(is_unsigned{}) - output[idx] = input_val & 0xfu; - else - // NOLINTNEXTLINE (hicpp-signed-bitwise) - output[idx] = static_cast(static_cast(input_val) << 4) >> 4; - - idx[Axis] += 1; - output[idx] = input_val >> 4; - }); -} - -} // namespace migraphx -#endif diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/vec.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/vec.hpp deleted file mode 100644 index ae453ff40..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/vec.hpp +++ /dev/null @@ -1,228 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_VEC_HPP -#define MIGRAPHX_GUARD_KERNELS_VEC_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { - -template -constexpr auto vec_size(vec) -{ - return index_constant{}; -} - -template -constexpr auto vec_size(T, ...) // NOLINT -{ - return index_constant<0>{}; -} - -template -constexpr auto vec_size() -{ - return decltype(vec_size(T{})){}; -} - -template -constexpr auto is_any_vec() -{ - if constexpr(sizeof...(Ts) == 0) - return false_type{}; - else - return bool_constant<((vec_size() + ...) > 0)>{}; -} - -template -constexpr auto vec_at(T x, I i) -{ - if constexpr(vec_size() == 0) - return x; - else - { - MIGRAPHX_ASSERT(i < vec_size()); - return x[i]; - } -} - -template -using vec_type = decltype(vec_at(T{}, 0)); - -template -constexpr auto common_vec_size() -{ - return fold([](auto x, auto y) { - if constexpr(x > y) - return x; - else - return y; - })(vec_size()...); -} - -// Bools can not be used as a vector type so convert it to uint8 -template -__device__ __host__ T* remove_bool(T* x) -{ - return x; -} - -inline __device__ __host__ uint8_t* remove_bool(bool* x) { return reinterpret_cast(x); } - -template -__device__ __host__ auto as_vec(T* x) -{ - if constexpr(N < 2) - return x; - else - return reinterpret_cast*>(x); -} - -template -using safe_vec = vec{}, uint8_t, T>, N>; - -template -constexpr auto vec_transform(Ts... xs) -{ - return [=](auto f) { - if constexpr(is_any_vec()) - { - using type = decltype(f(vec_at(xs, 0)...)); - constexpr auto size = common_vec_size(); - safe_vec result = {0}; - for(int i = 0; i < size; i++) - result[i] = f(vec_at(xs, i)...); - return result; - } - else - { - return f(xs...); - } - }; -} - -// Return a vector type of N from index i in another larger vector -// N will be 2 for half2 packing -template -constexpr vec, N> vec_packed_at(T x, I i) -{ - if constexpr(vec_size() == 0) - return vec{x}; - else - { - MIGRAPHX_ASSERT((i + N) <= vec_size()); - vec, N> result = {0}; - for(int j = 0; j < N; j++) - { - result[j] = x[i + j]; - } - return result; - } -} - -template -constexpr auto vec_packed_transform(Ts... xs) -{ - return [=](auto f) { - if constexpr(is_any_vec()) - { - using type = vec_type(xs, 0)...))>; - constexpr auto size = common_vec_size(); - safe_vec result = {0}; - for(int i = 0; i < size / N; i++) - { - // Call the function with packed vectors - safe_vec r = f(vec_packed_at(xs, i * N)...); - // Copy the packed vectors to the result - for(int j = 0; j < N; j++) - result[i * N + j] = r[j]; - } - return result; - } - else - { - return f(xs...); - } - }; -} - -template -constexpr auto vec_reduce(T x, Op op) -{ - if constexpr(vec_size() < 2) - return vec_type{x}; - else - { - vec_type result = x[0]; - for(int i = 1; i < vec_size(); i++) - result = op(result, x[i]); - return result; - } -} - -template -constexpr auto vec_generate(F f) -{ - using type = decltype(f(_c<0>)); - return sequence_c([&](auto... is) { return safe_vec{f(is)...}; }); -} - -template -struct implicit_conversion_op -{ - T x; - - template - constexpr operator vec() const - { - if constexpr(vec_size() == 0) - { - return x; - } - else - { - static_assert(vec_size() == N, "Vector mismatch size"); - return __builtin_convertvector(x, vec); - } - } - - template - constexpr operator U() const - { - return static_cast(x); - } -}; - -template -constexpr implicit_conversion_op implicit_conversion(T x) -{ - return {x}; -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_VEC_HPP diff --git a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/vectorize.hpp b/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/vectorize.hpp deleted file mode 100644 index b456b5c6e..000000000 --- a/docker/rocm/migraphx/targets/gpu/kernels/include/migraphx/kernels/vectorize.hpp +++ /dev/null @@ -1,263 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_KERNELS_VECTORIZE_HPP -#define MIGRAPHX_GUARD_KERNELS_VECTORIZE_HPP - -#include -#include - -namespace migraphx { - -template -constexpr auto tensor_vec_size() -{ - return vec_size(); -} - -template -constexpr auto tensor_vec_size(T) -{ - return tensor_vec_size(); -} - -template -constexpr auto shape_step(Shape s, Axis) -{ - static_assert(N > 0, "Vector size must be non-zero"); - return sequence(s.lens.size(), [&](auto... is) { - auto lens = transform(s.lens, index_ints{}, [&](auto i, auto j) { - constexpr auto axis = Axis::to(); - MIGRAPHX_ASSERT(i != 0); - MIGRAPHX_ASSERT(j != axis or i % N == 0); - if(j == axis) - return i / N; - else - return i; - }); - auto strides = transform(s.strides, index_ints{}, [&](auto i, auto j) { - constexpr auto axis = Axis::to(); - // If stride of the axis is zero then we dont need to adjust the other strides - if(Shape{}.strides[axis] == 0) - return i; - MIGRAPHX_ASSERT(j == axis or i % N == 0); - if(j == axis) - return i; - else - return i / N; - }); - MIGRAPHX_ASSERT(make_shape(lens, strides).elements() * N == s.elements()); - MIGRAPHX_ASSERT(strides[Axis{}] == 0 or - make_shape(lens, strides).element_space() * N == s.element_space()); - return make_shape(lens, strides); - }); -} - -template -__device__ __host__ auto as_vec(T x, Axis axis) -{ - if constexpr(N < 2) - return x; - else - return make_tensor_view(as_vec(remove_bool(x.data())), - shape_step(x.get_shape(), axis)); -} - -template -constexpr auto tensor_step(T x, Axis axis) -{ - if constexpr(N < 2) - { - return x; - } - else - { - constexpr auto s = decltype(x.get_shape()){}; - MIGRAPHX_ASSERT(s.strides[axis] == 0); - return make_tensor_view(x.data(), shape_step(s, axis)); - } -} - -template -__device__ __host__ auto as_vec(IntegralConstant ic, T&& x) -{ - return as_vec(x); -} - -template -constexpr index_int find_vector_axis_c(Shape s) -{ - // Find the fastest axis that is not broadcasted - index_int axis = 0; - for(index_int i = 1; i < s.lens.size(); i++) - { - if(s.strides[i] == 0) - continue; - if(s.strides[axis] == 0 or - pack_compare(less{}, pack(s.strides[i], s.lens[i]), pack(s.strides[axis], s.lens[axis]))) - axis = i; - } - return axis; -} - -template -constexpr index_int find_vector_axis_c(Shapes... ss) -{ - const bool all_broadcasted = (ss.broadcasted() and ...); - index_int axis = 0; - bool b = false; - by([&](auto s) { - if(b) - return; - // Skip broadcasted shapes if there are shapes not broadcasted - if(not all_broadcasted and s.broadcasted()) - return; - axis = find_vector_axis_c(s); - if(s.strides[axis] == 1) - b = true; - })(ss...); - if(not b) - return -1; - return axis; -} - -template -constexpr auto find_vector_axis(Shapes...) -{ - return _c; -} - -template -constexpr auto is_vectorizable_c(Axis axis, Shapes... ss) -{ - return ((axis < ss.lens.size() and ss.lens[axis] % N == 0 and - // Only vectorize broadcasted types with stride 0, since this causes issues in the - // preloader - ((not ss.broadcasted() and ss.strides[axis] == 1) or ss.strides[axis] == 0)) and - ...); -} - -template -constexpr auto is_vectorizable(Axis, Shapes...) -{ - return _c(Axis::to(), Shapes{}...)>; -} - -template -constexpr auto find_vectorize_size(P pred) -{ - if constexpr(decltype(pred(_c<4>)){}) - return _c<4>; - else if constexpr(decltype(pred(_c<2>)){}) - return _c<2>; - else - return _c<1>; -} - -template -__host__ __device__ auto auto_vectorize(T x) -{ - if constexpr(tensor_vec_size() == 0) - { - constexpr auto axis = find_vector_axis(x.get_shape()); - constexpr auto n = - find_vectorize_size([&](auto i) { return is_vectorizable(axis, x.get_shape()); }); - return as_vec(x, axis); - } - else - { - return x; - } -} - -template -inline __device__ __host__ auto auto_vectorize_impl(F f, Ts... xs) -{ - // TODO: Just check there a single axis of 1 - constexpr bool packed_or_broadcasted = - ((xs.get_shape().packed() or xs.get_shape().broadcasted()) and ...); - if constexpr(packed_or_broadcasted) - { - constexpr auto axis = decltype(find_vector_axis(xs.get_shape()...)){}; - constexpr auto n = find_vectorize_size( - [&](auto i) { return is_vectorizable(axis, xs.get_shape()...); }); - by( - [&](auto x) { - constexpr auto s = decltype(x.get_shape()){}; - if constexpr(axis < s.strides.size()) - { - MIGRAPHX_ASSERT(s.strides[axis] == 0 or s.strides[axis] == 1); - MIGRAPHX_ASSERT(s.lens[axis] > 0); - MIGRAPHX_ASSERT(n == 1 or s.lens[axis] % n == 0); - if constexpr(s.strides[axis] == 0) - return tensor_step(x, axis); - else - return as_vec(x, axis); - } - else - { - return x; - } - }, - f)(xs...); - } - else - { - f(xs...); - } -} - -inline __device__ __host__ auto auto_vectorize() -{ - return make_transform([](auto f, auto... xs) { auto_vectorize_impl(f, xs...); }); -} - -template -__device__ __host__ auto vectorize_tensor(T x) -{ - constexpr auto shape = get_shape_c{}; - if constexpr(shape.lens[Axis] == 1) - return x; - else if constexpr(shape.strides[Axis] == 0) - return tensor_step(x, _c); - else - return as_vec(x, _c); -} - -template -__device__ __host__ auto vectorize() -{ - return make_transform([](auto f, auto... xs) { - if constexpr(N < 2) - { - f(xs...); - } - else - { - f(vectorize_tensor(xs)...); - } - }); -} - -} // namespace migraphx -#endif // MIGRAPHX_GUARD_KERNELS_VECTORIZE_HPP diff --git a/docker/rocm/migraphx/targets/gpu/logsoftmax.cpp b/docker/rocm/migraphx/targets/gpu/logsoftmax.cpp deleted file mode 100644 index 63fc5eb5e..000000000 --- a/docker/rocm/migraphx/targets/gpu/logsoftmax.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_logsoftmax::compute_shape(const std::vector& inputs) const -{ - check_shapes{inputs, *this}.has(2).standard(); - return op.normalize_compute_shape({inputs.at(0)}); -} - -argument -hip_logsoftmax::compute(context& ctx, const shape&, const std::vector& args) const -{ - auto n_dim = args.front().get_shape().lens().size(); - auto tuned_axis = tune_axis(n_dim, op.axis, op.name()); - device::logsoftmax(ctx.get_stream().get(), args.back(), args.front(), tuned_axis); - return args.back(); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/loop.cpp b/docker/rocm/migraphx/targets/gpu/loop.cpp deleted file mode 100644 index ad5fc210c..000000000 --- a/docker/rocm/migraphx/targets/gpu/loop.cpp +++ /dev/null @@ -1,126 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_loop::compute_shape(std::vector inputs, std::vector mods) const -{ - auto input_num = (inputs.size() - 2) / 2; - inputs.erase(inputs.begin() + input_num, inputs.end()); - return op.compute_shape(inputs, std::move(mods)); -} - -struct gpu_loop -{ - int64_t max_iterations = 0; - - template - void copy(context& ctx, const argument& src, T& dst) const - { - argument arg_dst{src.get_shape(), &dst}; - copy_from_gpu(ctx, src, arg_dst); - } - - template - void copy(context& ctx, T src, const argument& dst) const - { - argument arg_src{dst.get_shape(), &src}; - copy_to_gpu(ctx, arg_src, dst); - } - - void append(const std::vector&, - const std::vector&, - const std::vector&, - int64_t, - int64_t) const - { - } - - void set_zero(context& ctx, const std::vector& concatenated_outputs, int iter) const - { - if(iter >= max_iterations) - return; - - auto elem_num = max_iterations - iter; - for(const auto& out : concatenated_outputs) - { - auto s = out.get_shape(); - auto size = s.bytes() / max_iterations; - auto lens = s.lens(); - lens[0] = elem_num; - shape ss{s.type(), lens}; - assert(ss.bytes() + iter * size <= out.get_shape().bytes()); - device::fill(ctx.get_stream().get(), argument(ss, out.data() + iter * size), 0); - } - } - - std::unordered_map get_output_params(const module& m) const - { - auto get_output_index = [](const std::string& name) { - std::string out_prefix = "#output_"; - auto loc = name.find(out_prefix); - if(loc != std::string::npos) - { - return std::stoi(name.substr(loc + out_prefix.size())); - } - - return -1; - }; - - const auto& param_names = m.get_parameter_names(); - std::unordered_map result; - for(const auto& name : param_names) - { - auto index = get_output_index(name); - if(index == -1) - continue; - result[name] = index; - } - - return result; - } -}; - -argument -hip_loop::compute(context& ctx, - const shape&, - const std::vector& args, - const std::vector& mods, - const std::function( - module_ref&, const std::unordered_map&)>& run) const -{ - return run_loop(gpu_loop{op.max_iterations}, op.scan_output_directions, ctx, args, mods, run); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/lowering.cpp b/docker/rocm/migraphx/targets/gpu/lowering.cpp deleted file mode 100644 index adba54661..000000000 --- a/docker/rocm/migraphx/targets/gpu/lowering.cpp +++ /dev/null @@ -1,599 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_HIPBLASLT_GEMM); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_MIOPEN_POOLING) - -struct miopen_apply -{ - module* mod = nullptr; - module_pass_manager* mpm = nullptr; - const lowering* pass = nullptr; - std::unordered_map> apply_map{}; - instruction_ref last{}; - bool offload_copy = false; - bool compute_fp32 = false; - - context& get_context() const - { - assert(pass != nullptr); - assert(pass->ctx != nullptr); - return *pass->ctx; - } - - void check_shape(shape x, instruction_ref i) - { - assert(x == i->get_shape()); - (void)x; - (void)i; - } - - void init() - { - assert(mod != nullptr); - assert(pass != nullptr); -#if MIGRAPHX_USE_ROCBLAS - compute_fp32 = get_compute_fp32_flag(); -#endif - offload_copy = (mod == mpm->get_root_module()) ? pass->offload_copy : false; - - add_extend_op("argmax"); - add_extend_op("argmin"); - add_extend_op("logsoftmax"); - add_extend_op("multinomial"); - add_extend_op("nonzero"); - add_extend_op("prefix_scan_sum"); - add_extend_op("reverse"); - add_extend_op("rnn_var_sl_last_output"); - add_extend_op("rnn_var_sl_shift_output"); - add_extend_op("rnn_var_sl_shift_sequence"); - add_extend_op("topk"); - add_generic_op("contiguous"); - add_pooling_op(); -#if MIGRAPHX_USE_MIOPEN - add_convolution_op("convolution"); - add_convolution_op("convolution_backwards"); - add_convolution_op("quant_convolution"); - add_extend_op("lrn"); -#endif -#if MIGRAPHX_USE_ROCBLAS or MIGRAPHX_USE_HIPBLASLT - add_gemm_op("dot"); - add_gemm_op("quant_dot"); -#endif - add_if_op(); - add_loop_op(); - add_neg_op(); - add_nms_op(); - add_lrn_op(); - add_convolution_backwards_op(); - add_select_module_op(); - add_reshape_lazy_op(); - add_group_query_attention_op(); - add_scan_slice_op(); - } - - void copy_params() const - { - if(not offload_copy) - return; - - for(auto ins : iterator_for(*mod)) - { - if(ins->name() != "@param") - continue; - - // parameter no outputs, no need to insert copy to gpu - if(ins->outputs().empty()) - continue; - - auto pos = std::next(ins); - auto a = insert_allocation(pos, ins->get_shape()); - auto c = mod->insert_instruction(pos, make_op("hip::copy_to_gpu"), ins, a); - mod->replace_instruction(ins, c); - } - - // return instruction - auto ret = std::prev(mod->end()); - if(ret->name() == "@return") - { - const auto& inputs = ret->inputs(); - - // each input of ret need to be copied from gpu to host, and replace - // output with copy output - for(const auto& in : inputs) - { - auto p_output = mod->insert_instruction(ret, make_op("hip::copy_from_gpu"), in); - instruction::replace_argument(ret, in, p_output); - } - } - // else branch to handle legacy program without the return instruction - else - { - mod->add_instruction(make_op("hip::copy_from_gpu"), ret); - } - } - - void apply() - { - init(); - for(auto it = mod->begin(); it != mod->end(); it++) - { - auto s = it->get_shape(); - auto attrs = it->get_operator().attributes(); - if(apply_map.count(it->name()) > 0) - { - check_shape(s, apply_map.at(it->name())(it)); - } - else if(has_compiler_for(it->name())) - { - check_shape(s, insert_precompile_op(it)); - } - else if(attrs.contains("target")) - { - check_shape(s, insert_custom_op(it, attrs)); - } - if(attrs.contains("prefill")) - { - insert_fill(it, attrs.at("prefill")); - } - } - copy_params(); - } - - void insert_fill(instruction_ref ins, value v) const - { - instruction_ref alloc = instruction::get_output_alias(ins, true); - if(alloc == ins) - return; - auto fill = mod->insert_instruction(ins, make_op("hip::fill", {{"value", v}}), alloc); - instruction::replace_argument(ins, alloc, fill); - } - - instruction_ref insert_custom_op(instruction_ref ins, const value& attrs) const - { - const auto& custom_op = ins->get_operator(); - if(attrs.at("target") == "cpu") - { - auto s = ins->get_shape(); - std::vector cpu_inputs; - auto inputs = ins->inputs(); - auto output = inputs.back(); - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) { - return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in); - }); - cpu_inputs.front() = - mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs); - auto cpu_out = mod->insert_instruction(ins, custom_op, cpu_inputs); - auto gpu_out = - mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output); - return mod->replace_instruction(ins, gpu_out); - } - return ins; - } - - instruction_ref insert_precompile_op(instruction_ref ins) const - { - auto output = insert_allocation(ins, ins->get_shape()); - std::vector refs = ins->inputs(); - refs.push_back(output); - - return mod->replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}), - refs, - ins->module_inputs()); - } - - instruction_ref insert_allocation(instruction_ref ins, const shape& s) const - { - return mod->insert_instruction(ins, make_op("allocate", {{"shape", to_value(s)}})); - } - -#if MIGRAPHX_USE_ROCBLAS or MIGRAPHX_USE_HIPBLASLT - template - void add_gemm_op(const std::string& name) - { - apply_map.emplace(name, [=](instruction_ref ins) { - std::vector refs = ins->inputs(); - assert(refs.size() == 2); - auto output = insert_allocation(ins, ins->get_shape()); - refs.push_back(output); -#if MIGRAPHX_USE_HIPBLASLT - if(not enabled(MIGRAPHX_ENABLE_HIPBLASLT_GEMM{}) or not hipblaslt_supported()) - { -#endif - return mod->replace_instruction( - ins, rocblas_gemm{Op{}, 1, 0, compute_fp32}, refs); -#if MIGRAPHX_USE_HIPBLASLT - } - std::string op_name = "gpu::hip_gemm"; - if(contains(name, "quant_")) - { - op_name = "gpu::hip_quant_gemm"; - } - operation gemm_op = make_op(op_name); - return mod->replace_instruction( - ins, - make_op("gpu::hipblaslt_op", {{"op", to_value(gemm_op)}}), - ins->inputs().at(0), - ins->inputs().at(1), - output); -#endif - }); - } -#endif - -#if MIGRAPHX_USE_MIOPEN - void add_convolution_op(const std::string& name) - { - apply_map.emplace(name, [=](instruction_ref ins) { - operation conv = make_op("gpu::" + name, {{"op", ins->get_operator().to_value()}}); - auto output = insert_allocation(ins, ins->get_shape()); - - return mod->replace_instruction(ins, - make_op("gpu::miopen_op", {{"op", to_value(conv)}}), - ins->inputs().at(0), - ins->inputs().at(1), - output); - }); - } -#endif - // add_generic_op just constructs the operator with no fields whereas add_extend_op copies over - // the fields Since it doesn't have fields its default constructed - - void add_generic_op(const std::string& name) { add_generic_op(name, "gpu::" + name); } - - void add_generic_op(const std::string& op_name, const std::string& gpu_name) - { - apply_map.emplace(op_name, [=](instruction_ref ins) { - auto output = insert_allocation(ins, ins->get_shape()); - std::vector refs = ins->inputs(); - refs.push_back(output); - - return mod->replace_instruction(ins, make_op(gpu_name), refs); - }); - } - - void add_extend_op(const std::string& name) { add_extend_op(name, "gpu::" + name); } - - void add_extend_op(const std::string& op_name, const std::string& gpu_name) - { - apply_map.emplace(op_name, [=](instruction_ref ins) { - auto&& op = ins->get_operator(); - auto output = insert_allocation(ins, ins->get_shape()); - std::vector refs = ins->inputs(); - refs.push_back(output); - - return mod->replace_instruction(ins, make_op(gpu_name, op.to_value()), refs); - }); - } - - static bool use_miopen_pooling(instruction_ref ins) - { - if(enabled(MIGRAPHX_DISABLE_MIOPEN_POOLING{}) or - not contains({shape::float_type, shape::half_type}, ins->get_shape().type())) - return false; - auto&& op = ins->get_operator(); - auto op_val = op.to_value(); - auto mode = op_val.at("mode").to(); - if(op_val.at("count_include_pad").to() and mode == op::pooling_mode::average) - return false; - if(mode == op::pooling_mode::lpnorm) - return false; - auto op_padding = op_val.at("padding").to_vector(); - auto kdims = ins->get_shape().lens().size() - 2; - return std::equal(op_padding.begin(), - op_padding.begin() + kdims, - op_padding.begin() + kdims, - op_padding.end()); - } - - void add_pooling_op() - { - apply_map.emplace("pooling", [=](instruction_ref ins) { - if(not use_miopen_pooling(ins)) - return insert_precompile_op(ins); -#if MIGRAPHX_USE_MIOPEN - auto output = insert_allocation(ins, ins->get_shape()); - std::vector refs = ins->inputs(); - auto&& op = ins->get_operator(); - refs.push_back(output); - return mod->replace_instruction(ins, make_op("gpu::pooling", op.to_value()), refs); -#else - return insert_precompile_op(ins); -#endif - }); - } - - // use 0 - input to represent neg - void add_neg_op() - { - apply_map.emplace("neg", [=](instruction_ref ins) { - auto s = ins->get_shape(); - std::vector zeros(s.elements(), 0.0f); - auto l0 = mod->add_literal(literal(s, zeros)); - auto output = insert_allocation(ins, s); - return mod->replace_instruction( - ins, make_op("gpu::sub"), l0, ins->inputs().front(), output); - }); - } - - // add input and output argument for the if operator - void add_if_op() - { - apply_map.emplace("if", [=](instruction_ref ins) { - std::vector inputs = ins->inputs(); - auto cpu_cond = - mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.front()); - auto sync_cond = mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_cond); - inputs.front() = sync_cond; - - return mod->replace_instruction(ins, ins->get_operator(), inputs, ins->module_inputs()); - }); - } - - // replace the loop operator with gpu_loop operator - void add_loop_op() - { - apply_map.emplace("loop", [=](instruction_ref ins) { - std::vector inputs = ins->inputs(); - // copy max_iter from gpu to cpu - auto cpu_max_iter = - mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.at(0)); - auto cpu_cond = - mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs.at(1)); - auto synced_max_iter = - mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_max_iter, cpu_cond); - inputs.at(0) = synced_max_iter; - inputs.at(1) = cpu_cond; - auto copy_inputs = inputs; - std::transform(copy_inputs.begin(), - copy_inputs.end(), - std::back_inserter(inputs), - [&](auto in) { return insert_allocation(ins, in->get_shape()); }); - - auto mod_args = ins->module_inputs(); - auto output = insert_allocation(ins, ins->get_shape()); - - const auto* sub_mod = mod_args.front(); - auto cond_out = insert_allocation(ins, sub_mod->get_output_shapes().front()); - - // add cond and mod outputs to the argument list - inputs.push_back(cond_out); - inputs.push_back(output); - - return mod->replace_instruction( - ins, make_op("gpu::loop", ins->get_operator().to_value()), inputs, mod_args); - }); - } - - void add_nms_op() - { - apply_map.emplace("nonmaxsuppression", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - std::vector cpu_inputs; - auto inputs = ins->inputs(); - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) { - return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in); - }); - cpu_inputs.front() = - mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs); - auto cpu_out = mod->insert_instruction(ins, ins->get_operator(), cpu_inputs); - auto gpu_out = - mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output); - return mod->replace_instruction(ins, gpu_out); - }); - } - - void add_lrn_op() - { - apply_map.emplace("lrn", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - std::vector cpu_inputs; - auto inputs = ins->inputs(); - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) { - return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in); - }); - cpu_inputs.front() = - mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs); - auto cpu_out = mod->insert_instruction(ins, ins->get_operator(), cpu_inputs); - auto gpu_out = - mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output); - return mod->replace_instruction(ins, gpu_out); - }); - } - - void add_convolution_backwards_op() - { - apply_map.emplace("convolution_backwards", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - std::vector cpu_inputs; - auto inputs = ins->inputs(); - std::transform( - inputs.begin(), inputs.end(), std::back_inserter(cpu_inputs), [&](auto in) { - return mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), in); - }); - cpu_inputs.front() = - mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_inputs); - auto cpu_out = mod->insert_instruction(ins, ins->get_operator(), cpu_inputs); - auto gpu_out = - mod->insert_instruction(ins, make_op("hip::copy_to_gpu"), cpu_out, output); - return mod->replace_instruction(ins, gpu_out); - }); - } - - /** - * Adds dynamic allocation for submodule output parameter. - */ - void add_select_module_op() - { - apply_map.emplace("select_module", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - std::vector inputs = ins->inputs(); - inputs.push_back(output); - return mod->replace_instruction(ins, ins->get_operator(), inputs, ins->module_inputs()); - }); - } - - /** - * Adds reshape lazy to reshape ops that can be aliased instead of copied. - * `gpu::contiguous` are added before and after the reshape; these contiguous - * instructions can be removed by the eliminate_contiguous pass. - */ - void add_reshape_lazy_op() - { - apply_map.emplace("reshape", [=](instruction_ref ins) { - std::vector before_contiguous_args = ins->inputs(); - auto before_alloc = insert_allocation(ins, std::prev(ins)->get_shape()); - before_contiguous_args.push_back(before_alloc); - auto before_contig = - mod->insert_instruction(ins, make_op("gpu::contiguous"), {before_contiguous_args}); - - auto new_lazy_reshape = mod->insert_instruction( - ins, - make_op("reshape_lazy", {{"dims", {ins->get_operator().to_value().at("dims")}}}), - before_contig); - - std::vector after_contiguous_args = {new_lazy_reshape}; - auto after_alloc = insert_allocation(new_lazy_reshape, new_lazy_reshape->get_shape()); - after_contiguous_args.push_back(after_alloc); - return mod->replace_instruction(ins, make_op("gpu::contiguous"), after_contiguous_args); - }); - } - - void add_group_query_attention_op() - { - apply_map.emplace("gpu::gqa_rotary_embedding", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - auto new_inputs = ins->inputs(); - new_inputs.push_back(output); - return mod->replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}), - new_inputs); - }); - - apply_map.emplace("gpu::concat_past_present", [=](instruction_ref ins) { - return mod->replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}), - ins->inputs()); - }); - - apply_map.emplace("gpu::compute_attention_probabilities", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - auto new_inputs = ins->inputs(); - new_inputs.push_back(output); - return mod->replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}), - new_inputs); - }); - - apply_map.emplace("gpu::gqa_softmax", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto inputs = ins->inputs(); - - auto new_inputs = ins->inputs(); - new_inputs.push_back(inputs.at(2)); - return mod->replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}), - new_inputs); - }); - - apply_map.emplace("gpu::compute_attention_scores", [=](instruction_ref ins) { - auto s = ins->get_shape(); - auto output = insert_allocation(ins, s); - auto new_inputs = ins->inputs(); - new_inputs.push_back(output); - return mod->replace_instruction( - ins, - make_op("gpu::precompile_op", {{"op", to_value(ins->get_operator())}}), - new_inputs); - }); - } - - void add_scan_slice_op() - { - apply_map.emplace("scan_slice", [=](instruction_ref ins) { - auto inputs = ins->inputs(); - auto cpu_idx = mod->insert_instruction(ins, make_op("hip::copy_from_gpu"), inputs[1]); - inputs[1] = mod->insert_instruction(ins, make_op("hip::sync_stream"), cpu_idx); - return mod->replace_instruction( - ins, mod->insert_instruction(ins, ins->get_operator(), inputs)); - }); - } -}; - -void lowering::apply(module_pass_manager& mpm) const -{ - miopen_apply{&mpm.get_module(), &mpm, this}.apply(); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/lrn.cpp b/docker/rocm/migraphx/targets/gpu/lrn.cpp deleted file mode 100644 index 2e99c208d..000000000 --- a/docker/rocm/migraphx/targets/gpu/lrn.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_MIOPEN -shape miopen_lrn::compute_shape(const std::vector& inputs) const -{ - check_shapes{inputs, *this}.has(2).not_broadcasted(); - return inputs.at(1); -} - -argument miopen_lrn::compute(context& ctx, - const shape& output_shape, - const std::vector& args) const -{ - float alpha = 1; - float beta = 0; - auto x_desc = make_tensor(args[0].get_shape()); - auto y_desc = make_tensor(output_shape); - miopenLRNForward(ctx.get_stream().get_miopen(), - ldesc.get(), - &alpha, - x_desc.get(), - args[0].implicit(), - &beta, - y_desc.get(), - args[1].implicit(), - false, - nullptr); - - return args[1]; -} - -void miopen_lrn::finalize(context&, const shape&, const std::vector&) -{ - ldesc = make_lrn(op); -} -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/mlir.cpp b/docker/rocm/migraphx/targets/gpu/mlir.cpp deleted file mode 100644 index 61e0325ac..000000000 --- a/docker/rocm/migraphx/targets/gpu/mlir.cpp +++ /dev/null @@ -1,1300 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef MIGRAPHX_MLIR -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if !defined(MLIR_MIGRAPHX_DIALECT_API_VERSION) || MLIR_MIGRAPHX_DIALECT_API_VERSION != 4 -#warning "Incompatible version of rocMLIR library used, disabling" -// Only undefine when not using cppcheck -#ifndef CPPCHECK -#undef MIGRAPHX_MLIR -#endif -#else -#include -#endif -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_TRACE_MLIR); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_TUNE_EXHAUSTIVE); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_TUNE_LIMIT); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_TUNING_DB); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_TUNING_CFG); -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_MLIR_ENABLE_SPLITK); - -#ifdef MIGRAPHX_MLIR -template // NOLINT -struct mlir_handle -{ - struct ptr - { - ptr() = default; - ptr(std::nullptr_t) {} - ptr(T x) : obj(x) {} - - std::intptr_t get_value() const - { - static_assert(sizeof(T) == sizeof(std::intptr_t), "MLIR Handle different size"); - return reinterpret_cast(obj); - } - - T get() const { return obj; } - - friend bool operator==(ptr x, ptr y) { return x.get_value() == y.get_value(); } - - friend bool operator!=(ptr x, ptr y) { return not(x == y); } - - explicit operator bool() const noexcept { return obj != ptr(); } - T obj{}; - }; - - struct deleter - { - using pointer = ptr; - - void operator()(pointer x) const - { - if(x != nullptr) - { - (void)f(x.obj); - } - } - }; - - mlir_handle() : handle(nullptr) {} - - mlir_handle(T p) : handle(ptr{p}) {} - - T get() const - { - return handle.get().get(); // NOLINT(readability-redundant-smartptr-get) - } - - T release() { return handle.release().get(); } - - private: - std::unique_ptr handle; -}; - -#define MIGRAPHX_MANAGE_MLIR_HANDLE(T, F) migraphx::gpu::mlir_handle // NOLINT - -using mlir_context = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirContext, mlirContextDestroy); -using mlir_thread_pool = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirLlvmThreadPool, mlirLlvmThreadPoolDestroy); -using mlir_dialect_registry = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirDialectRegistry, - mlirDialectRegistryDestroy); -using mlir_module = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirModule, mlirModuleDestroy); -using mlir_operation = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirOperation, mlirOperationDestroy); -using mlir_op_printing_flags = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirOpPrintingFlags, - mlirOpPrintingFlagsDestroy); -using mlir_region = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirRegion, mlirRegionDestroy); -using mlir_block = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirBlock, mlirBlockDestroy); -using mlir_pass_manager = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirPassManager, mlirPassManagerDestroy); -using mlir_tuning_table = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirRockTuningTable, - mlirRockTuningTableDestroy); -using mlir_tuning_space = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirRockTuningSpace, - mlirRockTuningSpaceDestroy); -using mlir_tuning_param = MIGRAPHX_MANAGE_MLIR_HANDLE(MlirRockTuningParam, - mlirRockTuningParamDestroy); - -std::string_view to_string_view(MlirStringRef s) { return {s.data, s.length}; } - -MlirStringRef make_mlir_string_ref(const std::string_view& s) -{ - return mlirStringRefCreate(s.data(), s.size()); -} - -template -void mlir_print(F f, T x, Printer printer) -{ - f( - x, - +[](MlirStringRef s, void* data) { - (*reinterpret_cast(data))(to_string_view(s)); - }, - &printer); -} - -template -void mlir_print(F f, T x, std::ostream& os) -{ - mlir_print(f, x, [&](auto s) { os << s; }); -} - -template -std::string mlir_print(F f, T x) -{ - std::stringstream ss; - mlir_print(f, x, [&](auto s) { ss << s; }); - return ss.str(); -} - -struct mlir_logger -{ - std::stringstream ss; - mlir_context* ctx; - std::optional id; - - mlir_logger() : ctx(nullptr), id(std::nullopt) {} - - mlir_logger(mlir_context* context) : ctx(context) - { - id = - mlirContextAttachDiagnosticHandler(ctx->get(), mlir_diagnostic_print_cb, this, nullptr); - } - - ~mlir_logger() - { - if(id.has_value()) - mlirContextDetachDiagnosticHandler(ctx->get(), *id); - } - - mlir_logger(const mlir_logger& other) = delete; - mlir_logger& operator=(const mlir_logger& other) = delete; - - mlir_logger(mlir_logger&& other) noexcept - : ss(std::move(other.ss)), ctx(other.ctx), id(other.id) - { - other.ctx = nullptr; - other.id = std::nullopt; - } - - mlir_logger& operator=(mlir_logger other) noexcept - { - std::swap(ss, other.ss); - std::swap(ctx, other.ctx); - std::swap(id, other.id); - return *this; - } - - std::string str() const { return ss.str(); } - - void clear() { ss = std::stringstream{}; } - - static MlirLogicalResult mlir_diagnostic_print_cb(MlirDiagnostic diag, void* logger); - - MlirLogicalResult handle(MlirDiagnostic diag); -}; - -MlirLogicalResult mlir_logger::mlir_diagnostic_print_cb(MlirDiagnostic diag, void* logger) -{ - return reinterpret_cast(logger)->handle(diag); -} - -MlirLogicalResult mlir_logger::handle(MlirDiagnostic diag) -{ - MlirDiagnosticSeverity sev = mlirDiagnosticGetSeverity(diag); - switch(sev) - { - case MlirDiagnosticSeverity::MlirDiagnosticError: ss << "Error: "; break; - case MlirDiagnosticSeverity::MlirDiagnosticWarning: ss << "Warning: "; break; - case MlirDiagnosticSeverity::MlirDiagnosticNote: ss << "Note: "; break; - case MlirDiagnosticSeverity::MlirDiagnosticRemark: ss << "Remark: "; break; - } - mlir_print(mlirDiagnosticPrint, diag, [&](auto s) { ss << s; }); - ss << std::endl; - for(intptr_t i = 0, e = mlirDiagnosticGetNumNotes(diag); i < e; ++i) - { - (void)handle(mlirDiagnosticGetNote(diag, i)); - } - return mlirLogicalResultSuccess(); -} - -struct mlir_program -{ - mlir_program() - : ctx(mlirContextCreateWithRegistry(get_dialect_registry().get(), - /*threadingEnable=*/false)), - location(mlirLocationUnknownGet(ctx.get())), - mmodule(mlirModuleCreateEmpty(location)), - logger(&ctx) - { - mlirContextSetThreadPool(ctx.get(), get_thread_pool().get()); - mlirContextLoadAllAvailableDialects(ctx.get()); - } - - static mlir_dialect_registry& get_dialect_registry() - { - static std::once_flag init_guard; - static mlir_dialect_registry the_registry; - // The MLIR registration functions (for dialects and passes) are not - // necessarily thread-safe and need to be executed exactly once - // (especially since they eventually call non-thread-safe LLVM - // initilizations). - std::call_once(init_guard, [&]() { - the_registry = mlirDialectRegistryCreate(); - mlirRegisterRocMLIRDialects(the_registry.get()); - mlirRegisterRocMLIRPasses(); - }); - return the_registry; - } - - static mlir_thread_pool& get_thread_pool() - { - // To save on overhead, we create one LLVM thread pool and reuse it - // across all MLIR contexts as recommended by MLIR upstream. - // Note that this is thread-safe as of C++11. - static mlir_thread_pool the_pool = mlirLlvmThreadPoolCreate(); - return the_pool; - } - - MlirType make_type(shape::type_t t) const - { - MlirType result; - shape::visit(t, [&](auto as) { - if(as.type_enum() == shape::float_type) - result = mlirF32TypeGet(ctx.get()); - else if(as.type_enum() == shape::half_type) - result = mlirF16TypeGet(ctx.get()); - else if(as.type_enum() == shape::bf16_type) - result = mlirBF16TypeGet(ctx.get()); - else if(as.type_enum() == shape::fp8e4m3fnuz_type) - result = mlirFloat8E4M3FNUZTypeGet(ctx.get()); - else if(as.type_enum() == shape::fp8e5m2fnuz_type) - result = mlirFloat8E5M2FNUZTypeGet(ctx.get()); - else if(as.type_enum() == shape::fp8e4m3fn_type) - result = mlirFloat8E4M3FNTypeGet(ctx.get()); - else if(as.type_enum() == shape::fp8e5m2_type) - result = mlirFloat8E5M2TypeGet(ctx.get()); - else if(as.type_enum() == shape::double_type) - result = mlirF64TypeGet(ctx.get()); - else if(as.is_integral()) - { - if(as.is_unsigned()) - { - result = mlirIntegerTypeUnsignedGet(ctx.get(), as.size() * 8); - } - else - { - result = mlirIntegerTypeSignedGet(ctx.get(), as.size() * 8); - } - } - else - MIGRAPHX_THROW("Unsupported type: " + std::to_string(as.type_enum())); - }); - return result; - } - - MlirType make_mlir_shaped(const shape& s) const - { - if(s.dynamic()) - MIGRAPHX_THROW("MLIR does not support dynamic shapes"); - std::vector lens(s.lens().begin(), s.lens().end()); - std::vector strides(s.strides().begin(), s.strides().end()); - return rocmlirMIXRShapedTypeGet( - lens.size(), lens.data(), strides.data(), make_type(s.type())); - } - - template - std::vector make_mlir_shapeds(const Range& r) - { - std::vector result; - std::transform(r.begin(), r.end(), std::back_inserter(result), [&](const auto& s) { - return make_mlir_shaped(s); - }); - return result; - } - - MlirType make_function_type(const std::vector& inputs, const std::vector& outputs) - { - auto in = make_mlir_shapeds(inputs); - auto out = make_mlir_shapeds(outputs); - return mlirFunctionTypeGet(ctx.get(), in.size(), in.data(), out.size(), out.data()); - } - - MlirIdentifier id(const std::string_view& s) const - { - return mlirIdentifierGet(ctx.get(), make_mlir_string_ref(s)); - } - - MlirAttribute attribute(std::int64_t i) const - { - return mlirIntegerAttrGet(mlirIntegerTypeGet(ctx.get(), 64), i); - } - MlirAttribute attribute(std::uint64_t i) const - { - if(i > (std::numeric_limits::max() / 2)) - MIGRAPHX_THROW("MLIR cant handle large integer values since they are ambiguous"); - return mlirIntegerAttrGet(mlirIntegerTypeGet(ctx.get(), 64), i); - } - MlirAttribute attribute(unsigned char i) const { return attribute(std::uint64_t(i)); } - MlirAttribute attribute(bool b) const { return mlirBoolAttrGet(ctx.get(), b ? 1 : 0); } - MlirAttribute attribute(double d) const - { - return mlirFloatAttrDoubleGet(ctx.get(), mlirF64TypeGet(ctx.get()), d); - } - MlirAttribute attribute(const std::string& s) const - { - return mlirStringAttrGet(ctx.get(), make_mlir_string_ref(s)); - } - MlirAttribute attribute(std::nullptr_t) const { return {}; } - template - MlirAttribute attribute(const std::vector& v) const - { - std::vector attributes; - attributes.reserve(v.size()); - std::transform(v.begin(), v.end(), std::back_inserter(attributes), [&](auto&& x) { - return attribute(x); - }); - return mlirArrayAttrGet(ctx.get(), attributes.size(), attributes.data()); - } - MlirAttribute attribute(const value& v) const - { - MlirAttribute attr; - v.visit_value([&](auto&& x) { attr = attribute(x); }); - return attr; - } - MlirAttribute attribute(const std::vector& v) const - { - if(v.empty()) - { - return mlirArrayAttrGet(ctx.get(), 0, nullptr); - } - if(not v.front().get_key().empty()) - { - std::vector attributes = name_attributes(v); - return mlirDictionaryAttrGet(ctx.get(), attributes.size(), attributes.data()); - } - else - { - std::vector attributes; - attributes.reserve(v.size()); - std::transform(v.begin(), v.end(), std::back_inserter(attributes), [&](auto&& x) { - return attribute(x); - }); - return mlirArrayAttrGet(ctx.get(), attributes.size(), attributes.data()); - } - } - - MlirAttribute attribute(MlirType t) const { return mlirTypeAttrGet(t); } - - MlirAttribute attribute(MlirAttribute a) const { return a; } - - template - MlirNamedAttribute name_attribute(const std::string_view& key, const T& x) const - { - MlirNamedAttribute attr; - attr.name = id(key); - attr.attribute = attribute(x); - return attr; - } - - using attribute_t = std::variant, - MlirType, - MlirAttribute>; - using named_attribute_t = std::pair; - - MlirNamedAttribute name_attribute(const named_attribute_t& na) const - { - return name_attribute(na.first, - std::visit([&](const auto& x) { return attribute(x); }, na.second)); - } - - std::vector - name_attributes(const std::vector& named_attrs) const - { - std::vector attributes; - attributes.reserve(named_attrs.size()); - std::transform(named_attrs.begin(), - named_attrs.end(), - std::back_inserter(attributes), - [&](const named_attribute_t& a) { return name_attribute(a); }); - return attributes; - } - - std::vector name_attributes(const value& v) const - { - std::vector attributes; - attributes.reserve(v.size()); - migraphx::transform_if( - v.begin(), - v.end(), - std::back_inserter(attributes), - [&](const value& x) { return not x.is_null(); }, - [&](const value& x) { return name_attribute(x.get_key(), x.without_key()); }); - return attributes; - } - - struct mlir_operation_state - { - mlir_operation_state(mlir_program& p, const std::string_view& name) - : prog(&p), op_state(mlirOperationStateGet(make_mlir_string_ref(name), p.location)) - { - } - - mlir_operation_state& add_attributes(const std::vector& named_attrs) - { - auto attributes = prog->name_attributes(named_attrs); - if(not attributes.empty()) - { - mlirOperationStateAddAttributes(&op_state, attributes.size(), attributes.data()); - } - return *this; - } - - mlir_operation_state& add_attribute_value(const value& v) - { - auto attributes = prog->name_attributes(v); - if(not attributes.empty()) - { - mlirOperationStateAddAttributes(&op_state, attributes.size(), attributes.data()); - } - return *this; - } - - mlir_operation_state& add_regions(std::vector rs) - { - regions = std::move(rs); - return *this; - } - - mlir_operation_state& add_region(mlir_region r) - { - regions.emplace_back(std::move(r)); - return *this; - } - - mlir_operation_state& add_results(const std::vector& outputs) - { - auto x = prog->make_mlir_shapeds(outputs); - if(not x.empty()) - { - mlirOperationStateAddResults(&op_state, x.size(), x.data()); - } - return *this; - } - - mlir_operation_state& add_operands(const std::vector& inputs) - { - if(not inputs.empty()) - { - mlirOperationStateAddOperands(&op_state, inputs.size(), inputs.data()); - } - return *this; - } - - mlir_operation create_operation() - { - std::vector mregions(regions.size()); - std::transform(regions.begin(), regions.end(), mregions.begin(), [](const auto& r) { - return r.get(); - }); - if(not mregions.empty()) - { - mlirOperationStateAddOwnedRegions(&op_state, mregions.size(), mregions.data()); - } - mlir_operation op(mlirOperationCreate(&op_state)); - // Release memory since mlir_operation owns it - for(auto& r : regions) - r.release(); - regions.clear(); - return op; - } - - mlir_program* prog; - MlirOperationState op_state; - std::vector regions = {}; - }; - - mlir_operation_state create_operation_state(const std::string_view& name) - { - return {*this, name}; - } - - std::vector insert(MlirBlock body, mlir_operation_state ops) - { - std::vector result; - mlir_operation op = ops.create_operation(); - auto weak_op = op.get(); - mlirBlockAppendOwnedOperation(body, op.release()); - - auto n = mlirOperationGetNumResults(weak_op); - result.reserve(n); - transform(range(n), std::back_inserter(result), [&](auto i) { - return mlirOperationGetResult(weak_op, i); - }); - return result; - } - - MlirBlock - insert(MlirBlock body, const module& m, std::unordered_map& ins_map) - { - auto names = m.get_parameter_names(); - std::sort(names.begin(), names.end()); - std::vector inputs; - std::transform(names.begin(), - names.end(), - std::back_inserter(inputs), - [&](const std::string& name) { return m.get_parameter_shape(name); }); - std::vector outputs = m.get_output_shapes(); - - std::vector arg_locs(inputs.size(), location); - auto body_inputs = make_mlir_shapeds(inputs); - mlir_region region = mlirRegionCreate(); - mlir_block fbody = mlirBlockCreate(body_inputs.size(), body_inputs.data(), arg_locs.data()); - MlirBlock result = fbody.get(); - mlirRegionAppendOwnedBlock(region.get(), fbody.release()); - - auto ops = create_operation_state("func.func"); - ops.add_attributes({{"function_type", make_function_type(inputs, outputs)}, - {"sym_name", sym_name}, - {"kernel", std::string("mixr")}, - {"arch", target_arch}, - {"num_cu", num_cu}}); - if(enabled(MIGRAPHX_MLIR_ENABLE_SPLITK{})) - { - ops.add_attributes({{"enable_splitk_for_tuning", mlirUnitAttrGet(ctx.get())}}); - } - ops.add_region(std::move(region)); - insert(body, std::move(ops)); - - for(auto i : range(names.size())) - ins_map[m.get_parameter(names[i])] = mlirBlockGetArgument(result, i); - return result; - } - - static bool is_reshape(const std::string& name) - { - return contains({"reshape", "lazy_reshape", "squeeze", "unsqueeze", "flatten"}, name); - } - - static std::string get_name(instruction_ref ins) - { - if(ins->name() == "@return") - return "func.return"; - if(ins->name() == "@literal") - return "migraphx.literal"; - if(ins->name() == "unpack_int4") - return "migraphx.unpack"; - if(is_reshape(ins->name())) - return "migraphx.reshape"; - return "migraphx." + ins->name(); - } - - static value get_operator_value(instruction_ref ins) - { - const operation& op = ins->get_operator(); - auto v = op.to_value(); - - // Reshape operator can have dim 0 or -1. - // Avoid passing those on to MLIR: - if(is_reshape(op.name())) - v = {{"dims", ins->get_shape().lens()}}; - - if(op.name() == "convolution" or op.name() == "quant_convolution") - { - // Adjust symetrical padding - if(v.at("padding").size() == v.at("stride").size()) - { - auto padding = v.at("padding"); - std::copy(padding.begin(), padding.end(), std::back_inserter(v.at("padding"))); - } - } - - if(op.name() == "unpack_int4") - v["axis"] = ins->get_shape().ndim() - 1; - - return v; - } - - static shape get_shape(instruction_ref ins) - { - if(ins->name() == "@return") - { - assert(ins->inputs().size() == 1); - return ins->inputs().front()->get_shape(); - } - return ins->get_shape(); - } - - static std::string get_symbol_name(const module& m) - { - return "mlir_" + gen::generate_name_from_ops(m); - } - - static void validate(const module& m) - { - if(m.begin() == m.end()) - MIGRAPHX_THROW("Empty module"); - auto last = std::prev(m.end()); - if(last->name() != "@return") - MIGRAPHX_THROW("Missing @return as last instruction."); - } - - void parse(const module& m) - { - validate(m); - sym_name = get_symbol_name(m); - auto mbody = mlirModuleGetBody(mmodule.get()); - std::unordered_map ins_map; - auto fbody = insert(mbody, m, ins_map); - - for(auto ins : iterator_for(m)) - { - if(ins->name() == "@param") - continue; - if(ins->name() == "contiguous") - { - ins_map[ins] = ins_map[ins->inputs().at(0)]; - continue; - } - auto name = get_name(ins); - auto ops = create_operation_state(name); - ops.add_attribute_value(get_operator_value(ins)); - if(ins->name() != "@return") - ops.add_results({get_shape(ins)}); - - if(ins->name() == "@literal") - { - literal r = ins->get_literal(); - auto sh = ins->get_shape(); - - MlirType shaped_type = make_mlir_shaped(sh); - MlirType tensor_type = rocmlirMIXRShapedTypeAsTensor(shaped_type); - MlirAttribute mlir_value_attr = - mlirDenseElementsAttrRawBufferGet(tensor_type, r.get_shape().bytes(), r.data()); - ops.add_attributes({{"value", mlir_value_attr}}); - } - - if(ins->name() == "convolution" or ins->name() == "dot") - { - pp = - problem_params{ins->get_operator(), to_shapes(ins->inputs()), ins->get_shape()}; - } - - std::vector inputs; - transform( - ins->inputs(), std::back_inserter(inputs), [&](auto i) { return ins_map.at(i); }); - ops.add_operands(inputs); - - auto outputs = insert(fbody, std::move(ops)); - if(ins->name() != "@return") - { - assert(outputs.size() == 1); - ins_map[ins] = outputs.front(); - } - } - } - - void run_high_level_pipeline() - { - mlir_pass_manager pm_front{mlirPassManagerCreate(ctx.get())}; - mlirMIGraphXAddHighLevelPipeline(pm_front.get()); - logger.clear(); - if(mlirLogicalResultIsFailure( - mlirPassManagerRunOnOp(pm_front.get(), mlirModuleGetOperation(mmodule.get())))) - { - std::string error = "Invalid MLIR created: " + logger.str(); - if(enabled(MIGRAPHX_TRACE_MLIR{})) - { - std::cout << error << std::endl; - } - MIGRAPHX_THROW(error); - } - } - - void run_backend_pipeline() - { - mlir_pass_manager pm_back{mlirPassManagerCreate(ctx.get())}; - mlirMIGraphXAddBackendPipeline(pm_back.get(), target_arch.c_str()); - logger.clear(); - const size_t trace = value_of(MIGRAPHX_TRACE_MLIR{}); - static std::mutex mutex; - auto mod_op = mlirModuleGetOperation(mmodule.get()); - if(trace >= 2) - { - const std::lock_guard lock(mutex); - std::cout << mlir_print(&mlirOperationPrint, mod_op) << std::endl; - } - - if(mlirLogicalResultIsFailure(mlirPassManagerRunOnOp(pm_back.get(), mod_op))) - { - std::string error = "MLIR backend compilation failed: " + logger.str(); - if(enabled(MIGRAPHX_TRACE_MLIR{})) - { - std::cout << error << std::endl; - } - MIGRAPHX_THROW(error); - } - } - - code_object_op compile(const value& solution) - { - // 1st pipeline to call - run_high_level_pipeline(); - if(solution.is_null()) - get_module_tuned(); - else - set_tuning(solution); - // 2nd pipeline to call - run_backend_pipeline(); - - code_object_op op{}; - op.symbol_name = sym_name; - op.code_object = get_binary(); - std::tie(op.global, op.local) = get_launch_params(); - return op; - } - - void set_gpu_properties(const context& migraphx_ctx) - { - const auto& device = migraphx_ctx.get_current_device(); - target_arch = device.get_device_name(); - num_cu = device.get_cu_count(); - } - - std::pair get_launch_params() const - { - uint32_t attrs[2]; - // returns block and grid sizes - mlirGetKernelAttrs(mmodule.get(), attrs); - std::size_t local = attrs[0]; - std::size_t global = local * attrs[1]; - return {global, local}; - } - - value::binary get_binary() const - { - size_t size = 0; - mlirGetBinary(mmodule.get(), &size, nullptr); - value::binary result(size); - if(mlirGetBinary(mmodule.get(), &size, reinterpret_cast(result.data()))) - return result; - MIGRAPHX_THROW("Failed to compile mlir program"); - } - - void set_tuning(const value& v) MIGRAPHX_TIDY_CONST - { - const auto* str = v.if_string(); - if(str == nullptr) - MIGRAPHX_THROW("mlir tuning solutions must be strings"); - if(not mlirRockTuningSetFromStr(mmodule.get(), make_mlir_string_ref(*str))) - MIGRAPHX_THROW("Failed setting tuning key: " + *str); - } - - tuning_config get_tuning_config(bool exhaustive) - { - tuning_config tc; - run_high_level_pipeline(); - auto tuning_mode = - exhaustive ? RocmlirTuningParamSetKindFull : RocmlirTuningParamSetKindQuick; - if(enabled(MIGRAPHX_MLIR_TUNE_EXHAUSTIVE{})) - tuning_mode = RocmlirTuningParamSetKindExhaustive; - mlir_tuning_space params{mlirRockTuningSpaceCreate(mmodule.get(), tuning_mode)}; - const auto limit = - value_of(MIGRAPHX_MLIR_TUNE_LIMIT{}, std::numeric_limits::max()); - for(auto i : range(std::min(limit, mlirRockTuningGetNumParams(params.get())))) - { - mlir_tuning_param param{mlirRockTuningParamCreate()}; - if(not mlirRockTuningParamGet(params.get(), i, param.get())) - MIGRAPHX_THROW("Incorrect mlir tuning parameter: " + std::to_string(i)); - std::array perf_key; - size_t perf_key_bytes = - mlirRockTuningParamToString(param.get(), perf_key.data(), perf_key.size()); - if(perf_key_bytes > perf_key.size()) - MIGRAPHX_THROW("Tuning perf key was " + std::to_string(perf_key_bytes) + - " bytes and thus too long"); - tc.solutions.emplace_back( - std::string(perf_key.begin(), perf_key.begin() + perf_key_bytes)); - } - std::array tuning_key; - size_t tuning_key_bytes = - mlirRockTuningGetKey(mmodule.get(), tuning_key.data(), tuning_key.size()); - if(tuning_key_bytes > tuning_key.size()) - MIGRAPHX_THROW("Tuning table key was " + std::to_string(tuning_key_bytes) + - " bytes and thus too long"); - tc.problem = std::string(tuning_key.begin(), tuning_key.begin() + tuning_key_bytes); - return tc; - } - - std::string get_tune_params(bool xdlops) const { return get_mlir_perf_for_conv(pp, xdlops); } - - // This function appends to tuning cfg file that could be - // used with rocMLIR tuning scripts. - void dump_tuning_cfg(const std::string& prob_config) const - { - std::string tuning_cfg_path = string_value_of(MIGRAPHX_MLIR_TUNING_CFG{}); - if(not tuning_cfg_path.empty()) - { - std::vector tokens = split_string(prob_config, '\t'); - std::string prob = tokens[2]; - - if(starts_with(prob, "conv")) - { - tuning_cfg_path += ".conv"; - } - else - { - tuning_cfg_path += ".gemm"; - } - std::ofstream tuning_cfg(tuning_cfg_path, std::ios::app); - prob = - trim(prob, [](unsigned char c) { return (c == '\0') or (std::isspace(c) != 0); }); - tuning_cfg << prob << std::endl; - } - } - - static std::pair load_tuning_table() - { - mlir_tuning_table tuning_table{mlirRockTuningTableCreate()}; - bool found_table = false; - std::string tuning_db_path = string_value_of(MIGRAPHX_MLIR_TUNING_DB{}); - if(not tuning_db_path.empty()) - { - std::ifstream tuning_db_tsv(tuning_db_path); - if(tuning_db_tsv) - { - found_table = true; - std::string line; - while(std::getline(tuning_db_tsv, line)) - { - std::vector tokens = split_string(line, '\t'); - std::string arch = tokens[0]; - std::string num_cu = tokens[1]; - std::string prob = tokens[2]; - std::string perf = tokens[3]; - std::string key = arch.append("\t").append(num_cu).append("\t").append(prob); - mlirRockTuningUpdateTable(tuning_table.get(), - make_mlir_string_ref(key), - make_mlir_string_ref(perf), - 1.0); - } - } - } - else - { - found_table = false; - std::cerr - << "WARNING: MLIR tuning db not found. Please set MIGRAPHX_MLIR_TUNING_DB for " - "optimal performance." - << std::endl; - } - return std::make_pair(std::move(tuning_table), found_table); - } - - bool get_module_tuned() const - { - static std::pair tuning_table = load_tuning_table(); - if(not mlirRockTuningSetFromTable(tuning_table.first.get(), mmodule.get())) - { - std::array prob_config; - size_t prob_config_bytes = - mlirRockTuningGetKey(mmodule.get(), prob_config.data(), prob_config.size()); - if(prob_config_bytes >= prob_config.size()) - { - std::cerr << "MLIR tuning key overflowed buffer, needed " << prob_config_bytes - << " bytes" << std::endl; - return false; - } - std::string prob_config_str(prob_config.begin(), - prob_config.begin() + prob_config_bytes); - if(tuning_table.second) - { - std::cerr << "NOTE: MLIR tuning table did not include a key for " << prob_config_str - << std::endl; - } - dump_tuning_cfg(prob_config_str); - return false; - } - return true; - } - - mlir_context ctx; - MlirLocation location; - mlir_module mmodule; - mlir_logger logger; - problem_params pp; - std::deque strings{}; - std::string target_arch = ""; - std::size_t num_cu = 0; - std::string sym_name; -}; - -bool is_reduce(const instruction& ins) { return contains(ins.name(), "reduce"); } - -static void rewrite_reduce(module& m) -{ - for(auto i : iterator_for(m)) - { - if(is_reduce(*i)) - { - auto reduce_op = i->get_operator().to_value(); - auto reduce_axes = reduce_op["axes"].to_vector(); - auto reduce_lens = i->get_shape().lens(); - auto in_shape = i->inputs().front()->get_shape(); - auto in_lens = in_shape.lens(); - assert(in_shape.standard()); - assert(reduce_lens.size() == in_lens.size()); - assert(std::adjacent_find( - reduce_axes.begin(), reduce_axes.end(), [](auto axis_1, auto axis_2) { - return axis_2 - axis_1 > 1; - }) == reduce_axes.end()); - - std::vector new_rsp_dims; - std::vector new_reduce_axes; - for(const auto axis : range(in_shape.ndim())) - { - if(reduce_lens[axis] == in_lens[axis]) - { - new_rsp_dims.push_back(in_lens[axis]); - } - else if(new_reduce_axes.empty()) - { - assert(reduce_lens[axis] == 1); - new_rsp_dims.push_back(-1); - new_reduce_axes.push_back(axis); - } - } - auto rsp_ins = m.insert_instruction( - i, migraphx::make_op("reshape", {{"dims", new_rsp_dims}}), i->inputs().front()); - auto collapsed_reduce = m.insert_instruction( - i, migraphx::make_op("reduce_sum", {{"axes", new_reduce_axes}}), rsp_ins); - auto rsp_back = m.insert_instruction( - i, migraphx::make_op("reshape", {{"dims", reduce_lens}}), collapsed_reduce); - m.replace_instruction(i, rsp_back); - } - } - migraphx::run_passes(m, {migraphx::dead_code_elimination{}}); -} - -bool is_module_fusible(const module& m, const context& migraphx_ctx, const value& solution) -{ - auto mm = m; - rewrite_reduce(mm); - mlir_program mp; - mp.set_gpu_properties(migraphx_ctx); - mp.parse(mm); - mp.run_high_level_pipeline(); - return mlirIsModuleFusible(mp.mmodule.get(), make_mlir_string_ref(*solution.if_string())); -} - -void adjust_param_shapes(module& m, const std::vector& inputs) -{ - auto names = m.get_parameter_names(); - std::sort(names.begin(), names.end()); - for(auto i : range(names.size())) - { - const auto& name = names[i]; - const auto& input = inputs[i]; - auto param = m.get_parameter(name); - assert(param->get_shape().standard()); - if(input.standard()) - continue; - auto new_param = m.add_parameter(name + ".0", input); - m.replace_instruction(param, new_param); - m.remove_instruction(param); - } -} - -void replace_params_with_literals(module& m, const std::vector& inputs) -{ - auto names = m.get_parameter_names(); - std::sort(names.begin(), names.end()); - for(auto i : range(names.size())) - { - const auto& name = names[i]; - const auto& input = inputs[i]; - if(input->name() != "@literal") - continue; - auto param = m.get_parameter(name); - auto lit = m.add_literal(input->get_literal()); - m.replace_instruction(param, lit); - m.remove_instruction(param); - } -} - -std::string dump_mlir(module m, const std::vector& inputs) -{ - const_module_ref mr = &m; - if(not inputs.empty()) - { - adjust_param_shapes(m, inputs); - } - rewrite_reduce(m); - mlir_program mp; - mp.parse(*mr); - auto mod_op = mlirModuleGetOperation(mp.mmodule.get()); - return mlir_print(&mlirOperationPrint, mod_op); -} - -static std::string compute_dump_name(const module& m, const std::string& ext) -{ - std::vector sizes; - for(auto ins : iterator_for(m)) - { - if(contains({"quant_convolution", "quant_dot", "convolution", "dot"}, ins->name())) - sizes.insert(sizes.end(), ins->inputs().begin(), ins->inputs().end()); - } - auto name = - mlir_program::get_symbol_name(m) + "_" + shape::to_sizes_string(to_shapes(sizes)) + ext; - replace_string_inplace(name, ", ", "_"); - replace_string_inplace(name, ":", "s"); - return name; -} - -void dump_mlir_to_file(module m, const std::vector& inputs, const fs::path& location) -{ - static std::mutex mutex; - const std::lock_guard lock(mutex); - - if(not inputs.empty()) - { - adjust_param_shapes(m, inputs); - } - rewrite_reduce(m); - - auto name = compute_dump_name(m, ".mlir"); - auto f = location / name; - std::cout << "Dumping MLIR file to: " << f << std::endl; - - mlir_program mp; - mp.parse(m); - auto mod_op = mlirModuleGetOperation(mp.mmodule.get()); - - std::string mlir_str = mlir_print(&mlirOperationPrint, mod_op); - - write_string(f, mlir_str); -} - -std::string dump_mlir(module m) { return dump_mlir(std::move(m), {}); } - -mlir_code_object compile_mlir(const context& migraphx_ctx, - module m, - const std::vector& in_shapes, - const value& solution) -{ - adjust_param_shapes(m, in_shapes); - rewrite_reduce(m); - const bool trace = enabled(MIGRAPHX_TRACE_MLIR{}); - - static std::mutex mutex; - if(trace) - { - const std::lock_guard lock(mutex); - std::cout << m << std::endl; - } - - mlir_program mp; - - mp.set_gpu_properties(migraphx_ctx); - mp.parse(m); - auto mod_op = mlirModuleGetOperation(mp.mmodule.get()); - if(trace) - { - const std::lock_guard lock(mutex); - std::cout << mlir_print(&mlirOperationPrint, mod_op) << std::endl; - } - auto co = mp.compile(solution); - - co.expected_inputs = in_shapes; - auto out_shapes = m.get_output_shapes(); - if(out_shapes.size() == 1) - { - co.output = m.get_output_shapes().front(); - } - else - { - co.output = shape{out_shapes}; - } - mlir_code_object mco; - mco.cop = co; - size_t num_prefill_args = mlirGetNumPrefillArgs(mp.mmodule.get()); - if(num_prefill_args > 0) - { - std::vector prefill_indices(num_prefill_args); - std::vector prefill_mlir_values(num_prefill_args); - mlirGetPrefillArgsInfo( - mp.mmodule.get(), prefill_indices.data(), prefill_mlir_values.data(), num_prefill_args); - std::vector prefill_values(prefill_mlir_values.size()); - std::transform(prefill_mlir_values.begin(), - prefill_mlir_values.end(), - prefill_values.begin(), - [](const auto& v) { - // mlir sets fill attribute as float but migx hip::fill operator only - // supports integer type. - // TODO: Need to add checks that it is indeed an integer. - double dv = mlirFloatAttrGetValueDouble(v); - return static_cast(dv); - }); - mco.prefill_indices = prefill_indices; - mco.prefill_values = prefill_values; - } - return mco; -} - -instruction_ref insert_mlir(module& m, - instruction_ref ins, - code_object_op co, - const std::vector& inputs) -{ - - std::vector refs; - std::size_t last = 0; - refs.reserve(inputs.size()); - std::copy(inputs.begin(), inputs.end(), std::back_inserter(refs)); - last = refs.size() - 1; - co.expected_inputs = to_shapes(refs); - co.output_arg = last; - return m.insert_instruction(ins, co, refs); -} - -tuning_config get_tuning_config_mlir(const context& migraphx_ctx, - module m, - const std::vector& inputs, - bool exhaustive) -{ - adjust_param_shapes(m, inputs); - rewrite_reduce(m); - mlir_program mp; - mp.set_gpu_properties(migraphx_ctx); - mp.parse(m); - auto tc = mp.get_tuning_config(exhaustive); - const bool trace = enabled(MIGRAPHX_TRACE_MLIR{}); - static std::mutex mutex; - if(trace) - { - const std::lock_guard lock(mutex); - std::cout << "Problem: " << tc.problem << std::endl; - auto mod_op = mlirModuleGetOperation(mp.mmodule.get()); - std::cout << mlir_print(&mlirOperationPrint, mod_op) << std::endl; - } - return tc; -} - -void dump_mlir_to_mxr(module m, - const std::vector& inputs, - const fs::path& location) -{ - static std::mutex mutex; - const std::lock_guard lock(mutex); - - adjust_param_shapes(m, to_shapes(inputs)); - replace_params_with_literals(m, inputs); - std::vector sizes; - for(auto ins : iterator_for(m)) - { - if(contains({"quant_convolution", "quant_dot", "convolution", "dot"}, ins->name())) - sizes.insert(sizes.end(), ins->inputs().begin(), ins->inputs().end()); - } - auto name = compute_dump_name(m, ".mxr"); - auto f = location / name; - std::cout << "Dumping MXR file to: " << f << std::endl; - save(program{std::move(m)}, f.string()); -} - -#else - -template -void use(T&) -{ -} - -std::string dump_mlir(module) { return {}; } - -std::string dump_mlir(module m, const std::vector& inputs) -{ - use(m); - use(inputs); - return {}; -} - -// Disabling clang-tidy warning on non-real useage. -// NOLINTBEGIN(performance-unnecessary-value-param) -mlir_code_object compile_mlir(const context&, module, const std::vector&, const value&) -{ - return {}; -} - -instruction_ref -// cppcheck-suppress funcArgNamesDifferent -insert_mlir(module& m, instruction_ref, code_object_op co, const std::vector&) -{ - use(co); - use(m); - return m.end(); -} - -tuning_config get_tuning_config_mlir(const context&, module, const std::vector&, bool) -{ - return {}; -} -// NOLINTEND(performance-unnecessary-value-param) - -#endif - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/multinomial.cpp b/docker/rocm/migraphx/targets/gpu/multinomial.cpp deleted file mode 100644 index 51e5c48b4..000000000 --- a/docker/rocm/migraphx/targets/gpu/multinomial.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_multinomial::compute_shape(std::vector inputs) const -{ - check_shapes{inputs, *this}.has(3).only_dims(2).standard(); - inputs.pop_back(); - return op.compute_shape(inputs); -} - -argument -hip_multinomial::compute(context& ctx, const shape&, const std::vector& args) const -{ - device::multinomial(ctx.get_stream().get(), args.back(), args.front(), args[1]); - return args.back(); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/no_device.cpp b/docker/rocm/migraphx/targets/gpu/no_device.cpp deleted file mode 100644 index 0ccdbac74..000000000 --- a/docker/rocm/migraphx/targets/gpu/no_device.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifdef __HIP_DEVICE_COMPILE__ -#error \ - "Device compilation not allowed for migraphx_gpu. Do not link with hip::device. Device code should go into migraphx_device or migraphx_kernels" -#endif diff --git a/docker/rocm/migraphx/targets/gpu/nonzero.cpp b/docker/rocm/migraphx/targets/gpu/nonzero.cpp deleted file mode 100644 index 0ff281f88..000000000 --- a/docker/rocm/migraphx/targets/gpu/nonzero.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_nonzero::compute_shape(std::vector inputs) const -{ - return op.compute_shape({inputs.front()}); -} - -argument hip_nonzero::compute(context& ctx, const shape&, const std::vector& args) const -{ - return device::nonzero(ctx.get_stream().get(), args.back(), args.front()); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/pack_args.cpp b/docker/rocm/migraphx/targets/gpu/pack_args.cpp deleted file mode 100644 index 2c3f41cf6..000000000 --- a/docker/rocm/migraphx/targets/gpu/pack_args.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -std::vector pack_args(const std::vector& args) -{ - std::vector kernargs; - for(auto&& arg : args) - { - std::size_t n = arg.size; - const auto* p = static_cast(arg.data); - // Insert padding - std::size_t padding = (arg.align - (kernargs.size() % arg.align)) % arg.align; - kernargs.insert(kernargs.end(), padding, 0); - kernargs.insert(kernargs.end(), p, p + n); - } - return kernargs; -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/perfdb.cpp b/docker/rocm/migraphx/targets/gpu/perfdb.cpp deleted file mode 100644 index bdad925db..000000000 --- a/docker/rocm/migraphx/targets/gpu/perfdb.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -namespace { - -std::string get_layout(const shape& s, std::string labels) -{ - auto result = labels; - auto p = find_permutation(s); - std::transform(p.begin(), p.end(), result.begin(), [&](auto i) { return labels[i]; }); - return "'" + result + "'"; -} - -std::string get_type(const shape& s) -{ - static const std::unordered_map m = { - {shape::float_type, "'FP32'"}, - {shape::half_type, "'FP16'"}, - {shape::double_type, "'FP64'"}, - {shape::int8_type, "'INT8'"}, - {shape::int32_type, "'INT32'"}, - }; - auto it = m.find(s.type()); - if(it == m.end()) - return "UNKNOWN"; - return it->second; -} - -std::string generate_miopen_config(const problem_params& pp) -{ - value v = pp.op.to_value(); - auto input = pp.inputs[0].lens(); - auto weights = pp.inputs[1].lens(); - auto padding = v["padding"].to_vector(); - auto stride = v["stride"].to_vector(); - auto dilation = v["dilation"].to_vector(); - if(padding.size() != stride.size()) - padding.erase(padding.begin() + padding.size() / 2, padding.end()); - return to_string_range({std::string{" C.in_channels="}, to_string(input[1]), - std::string{" AND C.in_h="}, to_string(input[2]), - std::string{" AND C.in_w="}, to_string(input[3]), - std::string{" AND C.fil_h="}, to_string(weights[2]), - std::string{" AND C.fil_w="}, to_string(weights[3]), - std::string{" AND C.out_channels="}, to_string(weights[0]), - std::string{" AND C.batchsize="}, to_string(input[0]), - std::string{" AND C.pad_h="}, to_string(padding[0]), - std::string{" AND C.pad_w="}, to_string(padding[2]), - std::string{" AND C.dilation_h="}, to_string(dilation[0]), - std::string{" AND C.dilation_w="}, to_string(dilation[1]), - std::string{" AND C.conv_stride_h="}, to_string(stride[0]), - std::string{" AND C.conv_stride_w="}, to_string(stride[1]), - std::string{" AND C.layout="}, get_layout(pp.inputs[0], "NCHW"), - std::string{" AND C.data_type="}, get_type(pp.inputs[0]), - std::string{" AND C.direction="}, std::string{"'F'"}}, - " "); -} - -auto query_miopen_db(const std::string& query) -{ - static std::mutex g_db_mutex; // NOLINT - const std::lock_guard lock(g_db_mutex); - - // TODO: Store db as a static variable - const auto dbpath = fs::path{"/opt"} / "rocm" / "share" / "miopen" / "db" / "miopen.db"; - // Check if db file exists. - std::ifstream dbs(dbpath); - if(dbs.is_open()) - { - dbs.close(); - } - else - { - std::vector> empty; - return empty; - } - - auto db = sqlite::read(dbpath); - return db.execute(query); -} - -} // namespace - -std::string get_mlir_perf_for_conv(const problem_params& pp, bool xdlops) -{ - std::string solver = xdlops ? "ConvMlirIgemmFwdXdlops" : "ConvMlirIgemmFwd"; - std::string query = "select P.* \ - from perf_db P, config C \ - where P.config = C.id AND \ - P.solver = '${solver}' AND \ - ${config}"; - - auto results = query_miopen_db( - interpolate_string(query, {{"config", generate_miopen_config(pp)}, {"solver", solver}})); - if(results.empty()) - return ""; - return results.front().at("params"); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/pooling.cpp b/docker/rocm/migraphx/targets/gpu/pooling.cpp deleted file mode 100644 index a6f86f077..000000000 --- a/docker/rocm/migraphx/targets/gpu/pooling.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_MIOPEN -shape miopen_pooling::compute_shape(const std::vector& inputs) const -{ - check_shapes{inputs, *this}.has(2).standard(); - std::vector pooling_input = {inputs.at(0)}; - check_shapes{pooling_input, *this}.max_ndims(5); - return op.normalize_compute_shape(pooling_input); -} - -inline void reshape_if_1d(shape& input) -{ - auto dims = input.lens(); - - if(dims.size() == 3) - { - std::vector new_dims = dims; - new_dims.insert(new_dims.begin() + 2, 1); - input = shape{input.type(), new_dims}; - } -} - -argument miopen_pooling::compute(context& ctx, - const shape& output_shape, - const std::vector& args) const -{ - shape x_shape = args[0].get_shape(); - shape y_shape = output_shape; - - reshape_if_1d(x_shape); - reshape_if_1d(y_shape); - - auto x_desc = make_tensor(x_shape); - auto y_desc = make_tensor(y_shape); - - float alpha = 1; - float beta = 0; - - miopenPoolingForward(ctx.get_stream().get_miopen(), - pd.get(), - &alpha, - x_desc.get(), - args[0].implicit(), - &beta, - y_desc.get(), - args[1].implicit(), - false, - nullptr, - 0); - - return args[1]; -} - -void miopen_pooling::finalize(context&, const shape&, const std::vector&) -{ - if(pd == nullptr) - pd = make_pooling(op); -} -#endif -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/prefuse_ops.cpp b/docker/rocm/migraphx/targets/gpu/prefuse_ops.cpp deleted file mode 100644 index f8a8f8375..000000000 --- a/docker/rocm/migraphx/targets/gpu/prefuse_ops.cpp +++ /dev/null @@ -1,400 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef MIGRAPHX_USE_COMPOSABLEKERNEL -#include -#endif -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_LAYERNORM_FUSION); - -namespace { - -template -struct layernorm_base -{ - float epsilon = 1e-12f; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.epsilon, "epsilon")); - } - shape compute_shape(std::vector inputs, std::vector mods) const - { - std::size_t nargs = N; - if(not mods.empty()) - { - auto* pm = mods.front(); - nargs += pm->get_parameter_names().size() - 1; - } - check_shapes{inputs, static_cast(*this)}.has(nargs); - auto s = inputs.front(); - auto t = s.type(); - if(not mods.empty()) - t = mods.front()->get_output_shapes().front().type(); - - // Scalar output if all inputs are scalar - if(inputs.front().elements() == 1 and - all_of(inputs, [](const auto& ss) { return ss.scalar(); })) - return inputs.front(); - auto l_s = shape::from_permutation( - t, s.lens(), find_permutation(std::vector(inputs.begin(), inputs.begin() + N))); - // just prelayernorm or preadd_layernorm - if(nargs <= N) - return l_s; - // else, layernorm + pointwise fusion, preserve layout of fused op - std::vector lp_s(inputs.begin() + N, inputs.end()); - lp_s.insert(lp_s.begin(), l_s); - return shape::from_permutation(t, s.lens(), find_permutation(lp_s)); - } -}; - -struct layernorm : layernorm_base -{ - - std::string name() const { return "gpu::prelayernorm"; } -}; -MIGRAPHX_REGISTER_OP(layernorm); - -struct add_layernorm : layernorm_base -{ - std::string name() const { return "gpu::preadd_layernorm"; } -}; -MIGRAPHX_REGISTER_OP(add_layernorm); - -struct find_layernorm -{ - auto matcher() const { return match::layernorm(); } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto x_ins = r.instructions["x"]; - float eps = 0; - if(contains(r.instructions, "eps")) - eps = r.instructions["eps"]->eval().at(); - - m.replace_instruction(ins, layernorm{eps}, x_ins); - } -}; - -struct find_add_layernorm -{ - auto matcher() const - { - return match::name("gpu::prelayernorm")( - match::args(match::name("add")(match::used_once()).bind("add"))); - } - - void apply(module& m, const match::matcher_result& r) const - { - auto ins = r.result; - auto add_ins = r.instructions["add"]; - auto op = any_cast(ins->get_operator()); - - m.replace_instruction(ins, add_layernorm{op.epsilon}, add_ins->inputs()); - } -}; - -struct pre_gemm_softmax_gemm : gemm_softmax_gemm -{ - std::string name() const { return "gpu::pre_gemm_softmax_gemm"; } -}; -MIGRAPHX_REGISTER_OP(pre_gemm_softmax_gemm); - -auto is_ck_gemm() -{ - return match::make_basic_pred_matcher([=](instruction_ref ins) { -#ifdef MIGRAPHX_USE_COMPOSABLEKERNEL - if(not enabled(MIGRAPHX_ENABLE_CK{})) - return false; - if(ins->name() != "dot") - return false; - if(not pre_gemm_softmax_gemm::is_ck_supported_type(ins->get_shape().type())) - return false; - return true; -#else - (void)ins; - return false; -#endif - }); -} - -auto is_test_gemm(bool enable_attention) -{ - return match::make_basic_pred_matcher([=](instruction_ref ins) { - if(ins->name() != "dot") - return false; - return enable_attention; - }); -} - -auto is_bias_supported() -{ - return match::make_basic_pred_matcher([=](instruction_ref) { -#ifdef MIGRAPHX_USE_COMPOSABLEKERNEL - return not enabled(MIGRAPHX_ENABLE_CK{}); -#else - return true; -#endif - }); -} - -struct find_gemm_softmax_gemm -{ - bool enable_attention = false; - - auto matcher() const - { - auto gemm1 = match::skip(match::name("contiguous"))(match::name("dot")( - match::any_of(is_ck_gemm(), is_test_gemm(enable_attention)).bind("gemm1"))); - auto mul = match::name("mul")( - match::nargs(2), match::either_arg(0, 1)(match::is_constant().bind("scale"), gemm1)); - auto where = match::name("where")(match::arg(2)(match::is_constant().bind("select_const")), - match::arg(1)(mul), - match::arg(0)(match::any().bind("select_cond"))); - auto add = - match::name("add")(is_bias_supported(), - match::nargs(2), - match::either_arg(0, 1)(match::none_of(mul).bind("bias"), mul)); - auto softmax = match::name("softmax")(match::arg(0)(match::any_of(mul, add, gemm1, where))) - .bind("softmax"); - - return match::name("dot")( - match::any_of(is_ck_gemm(), is_test_gemm(enable_attention)).bind("gemm2"))( - match::arg(0)(softmax)); - } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto gemm2_ins = r.instructions["gemm2"]; - auto gemm1_ins = r.instructions["gemm1"]; - - float scale = 1.0; - if(contains(r.instructions, "scale")) - { - auto scale_lit = r.instructions["scale"]; - // CK only supports single-valued scale - scale_lit->eval().visit([&](const auto s) { - // CK only supports single-valued scale - if(not std::all_of( - s.begin() + 1, s.end(), [&](auto v) { return float_equal(v, s.front()); })) - return; - scale = s.front(); - }); - } - - auto inputs = gemm1_ins->inputs(); // A, B - if(contains(r.instructions, "select_cond")) - { - inputs.push_back(r.instructions["select_cond"]); - inputs.push_back(r.instructions["select_const"]); - } - if(contains(r.instructions, "bias")) - { - inputs.push_back(r.instructions["bias"]); - } - - inputs.push_back(gemm2_ins->inputs().back()); // B1 - - mpm.get_module().replace_instruction( - ins, pre_gemm_softmax_gemm{gemm2_ins->get_operator(), scale}, inputs); - } -}; - -struct gpu_compute_attention_probabilities : op::group_query_attention -{ - std::string name() const { return "gpu::compute_attention_probabilities"; } - - shape compute_shape(std::vector inputs) const - { - auto query_lens = inputs.front().lens(); - auto present_kv_seqlen = inputs.at(1).lens().at(2); - std::vector output_lens{ - query_lens.at(0), num_heads, query_lens.at(2), present_kv_seqlen}; - shape output_shape{inputs.front().type(), output_lens}; - return output_shape; - } -}; -MIGRAPHX_REGISTER_OP(gpu_compute_attention_probabilities); - -struct gpu_compute_attention_scores : op::group_query_attention -{ - std::string name() const { return "gpu::compute_attention_scores"; } - - shape compute_shape(std::vector inputs) const - { - auto query_lens = inputs.front().lens(); - std::size_t q_hidden_size = - (query_lens[1] * query_lens[3] * num_heads) / (num_heads + 2 * kv_num_heads); - std::vector output_lens{query_lens.at(0), query_lens.at(2), q_hidden_size}; - shape output_shape{inputs.front().type(), output_lens}; - return output_shape; - } -}; -MIGRAPHX_REGISTER_OP(gpu_compute_attention_scores); - -struct gpu_gqa_rotary_embedding : op::group_query_attention -{ - std::string name() const { return "gpu::gqa_rotary_embedding"; } - - shape compute_shape(std::vector inputs) const { return inputs.front(); } -}; -MIGRAPHX_REGISTER_OP(gpu_gqa_rotary_embedding); - -struct gpu_gqa_softmax : op::group_query_attention -{ - std::string name() const { return "gpu::gqa_softmax"; } - - shape compute_shape(std::vector inputs) const { return inputs.at(2); } -}; -MIGRAPHX_REGISTER_OP(gpu_gqa_softmax); - -struct gpu_concat_past_present : op::group_query_attention -{ - std::string name() const { return "gpu::concat_past_present"; } - - shape compute_shape(std::vector inputs) const { return inputs[0]; } -}; -MIGRAPHX_REGISTER_OP(gpu_concat_past_present); - -struct find_group_query_attention -{ - auto matcher() const { return match::name("group_query_attention"); } - - void apply(module_pass_manager& mpm, const match::matcher_result& r) const - { - auto ins = r.result; - auto inputs = ins->inputs(); - auto v = ins->get_operator().to_value(); - - auto num_heads = v.at("num_heads").to(); - auto kv_num_heads = v.at("kv_num_heads").to(); - auto do_rotary = v.at("do_rotary").to(); - auto local_window_size = v.at("local_window_size").to(); - auto rotary_interleaved = v.at("rotary_interleaved").to(); - auto scale = v.at("scale").to(); - - auto q_shape = inputs[0]->get_shape(); - auto q_lens = q_shape.lens(); - const std::size_t batch_size = q_lens[0]; - const std::size_t sequence_length = q_lens[1]; - std::size_t q_hidden_size = q_lens[2]; - std::size_t head_size = q_hidden_size / (num_heads + 2 * kv_num_heads); - - std::vector bsnh{ - batch_size, sequence_length, num_heads + 2 * kv_num_heads, head_size}; - - auto transposed_qkv = mpm.get_module().insert_instruction( - ins, make_op("reshape", {{"dims", bsnh}}), inputs.at(0)); - - transposed_qkv = mpm.get_module().insert_instruction( - ins, make_op("transpose", {{"permutation", {0, 2, 1, 3}}}), transposed_qkv); - - auto rotary_qkv = transposed_qkv; - if(do_rotary) - { - std::vector rotary_inputs{ - transposed_qkv, inputs.at(5), inputs.at(7), inputs.at(8)}; - rotary_qkv = - mpm.get_module().insert_instruction(ins, - gpu_gqa_rotary_embedding{do_rotary, - kv_num_heads, - local_window_size, - num_heads, - rotary_interleaved, - scale}, - rotary_inputs); - } - - auto pres_k = inputs.at(3); - auto pres_v = inputs.at(4); - std::vector concat_inputs{rotary_qkv, pres_k, pres_v, inputs.at(5)}; - - auto concat = mpm.get_module().insert_instruction( - ins, - gpu_concat_past_present{ - do_rotary, kv_num_heads, local_window_size, num_heads, rotary_interleaved, scale}, - concat_inputs); - auto id = - mpm.get_module().insert_instruction(ins, make_op("identity"), concat, pres_k, pres_v); - - std::vector attn_probs_inputs{id, pres_k, pres_v, inputs.at(5)}; - auto attn_probs = mpm.get_module().insert_instruction( - ins, - gpu_compute_attention_probabilities{ - do_rotary, kv_num_heads, local_window_size, num_heads, rotary_interleaved, scale}, - attn_probs_inputs); - - std::vector softmax_inputs{rotary_qkv, pres_k, attn_probs, inputs.at(5)}; - auto softmax = mpm.get_module().insert_instruction( - ins, - gpu_gqa_softmax{ - do_rotary, kv_num_heads, local_window_size, num_heads, rotary_interleaved, scale}, - softmax_inputs); - std::vector new_inputs{rotary_qkv, pres_k, pres_v, inputs.at(5), softmax}; - - auto get_tuple_elm_0 = std::next(ins); - auto get_tuple_elm_1 = std::next(get_tuple_elm_0); - auto get_tuple_elm_2 = std::next(get_tuple_elm_1); - mpm.get_module().replace_instruction(get_tuple_elm_2, pres_v); - mpm.get_module().replace_instruction(get_tuple_elm_1, pres_k); - mpm.get_module().replace_instruction( - get_tuple_elm_0, - gpu_compute_attention_scores{ - do_rotary, kv_num_heads, local_window_size, num_heads, rotary_interleaved, scale}, - new_inputs); - } -}; - -} // namespace - -void prefuse_ops::apply(module_pass_manager& mpm) const -{ - if(not enabled(MIGRAPHX_DISABLE_LAYERNORM_FUSION{})) - { - match::find_matches(mpm.get_module(), find_layernorm{}); - mpm.run_pass(dead_code_elimination{}); - match::find_matches(mpm.get_module(), find_add_layernorm{}); - } - match::find_matches(mpm, find_gemm_softmax_gemm{enable_attention}); - match::find_matches(mpm, find_group_query_attention{}); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/prepare_reduce.cpp b/docker/rocm/migraphx/targets/gpu/prepare_reduce.cpp deleted file mode 100644 index bd5abd42b..000000000 --- a/docker/rocm/migraphx/targets/gpu/prepare_reduce.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct parallel_reduce -{ - operation op; - - template - static auto reflect(Self& self, F f) - { - return pack(f(self.op, "op")); - } - - std::string name() const { return "gpu::parallel_reduce"; } - - shape compute_shape(const std::vector& inputs) const - { - std::vector result; - std::transform(inputs.begin(), inputs.end(), std::back_inserter(result), [&](auto input) { - return op.compute_shape({input}); - }); - return shape{result}; - } -}; -MIGRAPHX_REGISTER_OP(parallel_reduce); - -namespace { - -std::vector find_reduce(module& m) -{ - std::vector result; - auto im = iterator_for(m); - std::copy_if(im.begin(), im.end(), std::back_inserter(result), [](auto ins) { - if(contains({"gpu::parallel_reduce", "reduce_mean"}, ins->name())) - return false; - return contains(ins->name(), "reduce"); - }); - return result; -} - -std::vector find_parallel_reduce(const std::vector& r) -{ - std::vector result; - auto ir = iterator_for(r); - transform_if( - ir.begin(), - ir.end(), - std::back_inserter(result), - [&](auto x) { - return std::none_of( - std::next(x), r.end(), [&](auto reduce) { return reaches(*x, reduce); }); - }, - [](auto x) { return *x; }); - return result; -} - -void fuse_reductions(module& m) -{ - auto rs = find_parallel_reduce(find_reduce(m)); - if(rs.size() < 2) - return; - // Only handle the same reduction operator for now - if(std::any_of(std::next(rs.begin()), rs.end(), [&](auto r) { - return rs.front()->name() != r->name(); - })) - return; - auto last = rs.front(); - auto op = last->get_operator(); - std::vector inputs; - std::transform(rs.begin(), rs.end(), std::back_inserter(inputs), [&](auto r) { - return r->inputs().front(); - }); - auto pr = m.insert_instruction(last, parallel_reduce{op}, inputs); - int i = 0; - for(auto r : rs) - { - m.replace_instruction(r, make_op("get_tuple_elem", {{"index", i}}), pr); - i++; - } - m.sort(); -} - -} // namespace - -void prepare_reduce::apply(module& m) const { fuse_reductions(m); } - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/problem_cache.cpp b/docker/rocm/migraphx/targets/gpu/problem_cache.cpp deleted file mode 100644 index 8eb25f3b8..000000000 --- a/docker/rocm/migraphx/targets/gpu/problem_cache.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_PROBLEM_CACHE) - -void problem_cache::load() -{ - auto pc_path = string_value_of(MIGRAPHX_PROBLEM_CACHE{}); - if(pc_path.empty()) - return; - if(not fs::exists(pc_path)) - { - std::cout << "Problem cache not found. Creating new file.\n"; - return; - } - from_value(from_json_string(read_string(pc_path)), cache); -} -void problem_cache::save() const -{ - auto pc_path = string_value_of(MIGRAPHX_PROBLEM_CACHE{}); - if(pc_path.empty()) - return; - write_string(pc_path, to_pretty_json_string(to_value(cache))); -} - -static value create_key(const std::string& name, const value& problem) -{ - return {{"name", name}, {"problem", problem}}; -} - -bool problem_cache::has(const std::string& name, const value& problem) const -{ - return contains(cache, create_key(name, problem)); -} - -void problem_cache::insert(const std::string& name, const value& problem, const value& solution) -{ - assert(not solution.is_null()); - cache[create_key(name, problem)] = solution; -} - -void problem_cache::mark(const std::string& name, const value& problem) -{ - cache.insert(std::make_pair(create_key(name, problem), value{})); -} - -optional problem_cache::get(const std::string& name, const value& problem) const -{ - auto it = cache.find(create_key(name, problem)); - if(it == cache.end()) - return nullopt; - return it->second; -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/reverse.cpp b/docker/rocm/migraphx/targets/gpu/reverse.cpp deleted file mode 100644 index ea70e3fbb..000000000 --- a/docker/rocm/migraphx/targets/gpu/reverse.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_reverse::compute_shape(std::vector inputs) const -{ - inputs.pop_back(); - return op.normalize_compute_shape(inputs); -} - -argument hip_reverse::compute(context& ctx, const shape&, const std::vector& args) const -{ - return device::reverse(ctx.get_stream().get(), args.back(), args[0], op.axes); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/rnn_variable_seq_lens.cpp b/docker/rocm/migraphx/targets/gpu/rnn_variable_seq_lens.cpp deleted file mode 100644 index e251716f9..000000000 --- a/docker/rocm/migraphx/targets/gpu/rnn_variable_seq_lens.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_rnn_var_sl_shift_output::compute_shape(std::vector inputs) const -{ - inputs.pop_back(); - return op.compute_shape(inputs); -} - -argument hip_rnn_var_sl_shift_output::compute(context& ctx, - const shape&, - const std::vector& args) const -{ - device::rnn_var_sl_shift_output(ctx.get_stream().get(), - args.back(), - args.at(0), - args.at(1), - (op.direction == op::rnn_direction::reverse)); - return args.back(); -} - -shape hip_rnn_var_sl_shift_sequence::compute_shape(std::vector inputs) const -{ - inputs.pop_back(); - return op.compute_shape(inputs); -} - -argument hip_rnn_var_sl_shift_sequence::compute(context& ctx, - const shape&, - const std::vector& args) const -{ - device::rnn_var_sl_shift_sequence(ctx.get_stream().get(), args.back(), args.at(0), args.at(1)); - return args.back(); -} - -shape hip_rnn_var_sl_last_output::compute_shape(std::vector inputs) const -{ - inputs.pop_back(); - return op.compute_shape(inputs); -} - -argument hip_rnn_var_sl_last_output::compute(context& ctx, - const shape&, - const std::vector& args) const -{ - device::rnn_var_sl_last_output(ctx.get_stream().get(), - args.back(), - args.at(0), - args.at(1), - (op.direction == op::rnn_direction::reverse)); - return args.back(); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/rocblas.cpp b/docker/rocm/migraphx/targets/gpu/rocblas.cpp deleted file mode 100644 index 8c06ad51f..000000000 --- a/docker/rocm/migraphx/targets/gpu/rocblas.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { -#if MIGRAPHX_USE_ROCBLAS -rocblas_handle_ptr create_rocblas_handle_ptr() -{ - // add a call to rocblas_initialize() to workaround a rocblas bug SWDEV-438929 - rocblas_initialize(); - rocblas_handle handle; - rocblas_create_handle(&handle); - return rocblas_handle_ptr{handle}; -} - -rocblas_handle_ptr create_rocblas_handle_ptr(hipStream_t s) -{ - rocblas_handle_ptr rb = create_rocblas_handle_ptr(); - rocblas_set_stream(rb.get(), s); - return rb; -} -#endif -bool get_compute_fp32_flag() -{ - const auto device_name = trim(split_string(get_device_name(), ':').front()); - return (starts_with(device_name, "gfx9") and device_name >= "gfx908"); -} - -bool rocblas_fp8_available() -{ -#if MIGRAPHX_USE_ROCBLAS -#ifndef MIGRAPHX_USE_ROCBLAS_FP8_API - return false; -#else - return gfx_has_fp8fnuz_intrinsics(); -#endif -#else - return false; -#endif -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/schedule_model.cpp b/docker/rocm/migraphx/targets/gpu/schedule_model.cpp deleted file mode 100644 index aa59a693f..000000000 --- a/docker/rocm/migraphx/targets/gpu/schedule_model.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -struct record_event -{ - std::size_t event = 0; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.event, "event")); - } - std::string name() const { return "gpu::record_event"; } - shape compute_shape(const std::vector&) const { return {}; } - - argument compute(context& ctx, const shape&, const std::vector&) const - { - ctx.get_stream().record(ctx.get_event(event)); - return {}; - } - - void finalize(context& ctx, const shape&, const std::vector&) const - { - ctx.create_events(event); - } -}; - -struct wait_event -{ - std::size_t event = 0; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.event, "event")); - } - std::string name() const { return "gpu::wait_event"; } - shape compute_shape(const std::vector&) const { return {}; } - - argument compute(context& ctx, const shape&, const std::vector&) const - { - ctx.get_stream().wait(ctx.get_event(event)); - return {}; - } -}; - -struct set_stream -{ - std::size_t stream = 0; - template - static auto reflect(Self& self, F f) - { - return pack(f(self.stream, "stream")); - } - std::string name() const { return "gpu::set_stream"; } - shape compute_shape(const std::vector&) const { return {}; } - - argument compute(context& ctx, const shape&, const std::vector&) const - { - ctx.set_stream(stream); - return {}; - } - void finalize(context& ctx, const shape&, const std::vector&) const - { - ctx.set_stream(stream); - } -}; - -MIGRAPHX_REGISTER_OP(record_event) -MIGRAPHX_REGISTER_OP(wait_event) -MIGRAPHX_REGISTER_OP(set_stream) - -std::size_t schedule_model::concurrency() const { return streams; } -void schedule_model::sched(module& m, instruction_ref ins, std::size_t n) const -{ - auto last_stream = std::find_if(std::make_reverse_iterator(ins), - std::make_reverse_iterator(m.begin()), - [&](auto&& i) { return i.name() == "gpu::set_stream"; }); - if(last_stream != std::make_reverse_iterator(m.begin())) - { - auto&& op = any_cast(last_stream->get_operator()); - // If the same stream was set earlier then skip - if(op.stream == n) - return; - } - m.insert_instruction(ins, set_stream{n}); -} - -void schedule_model::wait(module& m, instruction_ref ins, std::size_t wait_id) const -{ - m.insert_instruction(ins, wait_event{wait_id}); -} -void schedule_model::record(module& m, instruction_ref ins, std::size_t wait_id) const -{ - m.insert_instruction(std::next(ins), record_event{wait_id}); -} - -static std::unordered_map create_weight_map() -{ - return {{"hip::load_literal", 0}, - {"hip::hip_allocate_memory", 0}, - {"hip::hip_load_memory", 0}, - {"hip::allocate", 0}, - {"gpu::convolution", 8}, - {"gpu::conv_bias_relu", 8}, - {"gpu::pooling", 4}, - {"gpu::gemm", 4}}; -} - -static const std::unordered_map& weight_map() -{ - static const std::unordered_map m = create_weight_map(); - return m; -} - -std::size_t schedule_model::weight(const operation& op) const -{ - if(weight_map().count(op.name()) == 0) - { - return 2; - } - return weight_map().at(op.name()); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/sync_device.cpp b/docker/rocm/migraphx/targets/gpu/sync_device.cpp deleted file mode 100644 index 4e8a176eb..000000000 --- a/docker/rocm/migraphx/targets/gpu/sync_device.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -void sync_device::apply(module& m) const -{ - auto last = std::prev(m.end()); - if(last->name() == "@return") - { - auto inputs = last->inputs(); - if(std::any_of(inputs.begin(), inputs.end(), [](auto i) { - return (i->name() == "hip::copy_from_gpu"); - })) - { - auto sync_in = m.insert_instruction(last, make_op("hip::sync_stream"), inputs); - if(not inputs.empty()) - { - m.replace_instruction(inputs.front(), sync_in); - } - } - } -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/target.cpp b/docker/rocm/migraphx/targets/gpu/target.cpp deleted file mode 100644 index bf2fc3e86..000000000 --- a/docker/rocm/migraphx/targets/gpu/target.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DISABLE_SCHEDULE_PASS) -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_NHWC) -#ifndef _WIN32 -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_CK) -#endif -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_ENABLE_HIPBLASLT_GEMM) - -std::vector target::get_passes(migraphx::context& gctx, const compile_options& options) const -{ - auto& ctx = any_cast(gctx); - ctx.set_exhaustive_tune_flag(options.exhaustive_tune); - ctx.load_problem_cache(); - std::set unsupported_types(shape::types().begin(), shape::types().end()); - unsupported_types.erase(shape::type_t::float_type); - unsupported_types.erase(shape::type_t::fp8e4m3fnuz_type); - unsupported_types.erase(shape::type_t::fp8e5m2fnuz_type); - unsupported_types.erase(shape::type_t::fp8e4m3fn_type); - unsupported_types.erase(shape::type_t::fp8e5m2_type); - unsupported_types.erase(shape::type_t::half_type); - unsupported_types.erase(shape::type_t::bool_type); - unsupported_types.erase(shape::type_t::int8_type); - unsupported_types.erase(shape::type_t::uint8_type); - unsupported_types.erase(shape::type_t::int32_type); - unsupported_types.erase(shape::type_t::tuple_type); - unsupported_types.erase(shape::type_t::bf16_type); - - // whiltelist supported Ops for the FP8 types - // different between fp8e4m3fnuz and OCP types because rocBLAS only has - // support for fp8e4m3fnuz - std::set unsupported_fp8e4m3fnuz_ops = {}; - if(not enabled(MIGRAPHX_ENABLE_HIPBLASLT_GEMM{}) and not gpu::rocblas_fp8_available()) - { - unsupported_fp8e4m3fnuz_ops.insert("dot"); - unsupported_fp8e4m3fnuz_ops.insert("quant_dot"); - } -#if MIGRAPHX_USE_MIOPEN - // MIOpen doesn't have support for fp8 pooling yet. - unsupported_fp8e4m3fnuz_ops.insert("pooling"); -#endif - if(not gpu::gfx_has_fp8fnuz_intrinsics()) - { - unsupported_fp8e4m3fnuz_ops.insert("convolution"); - unsupported_fp8e4m3fnuz_ops.insert("quant_convolution"); - } - // add all device kernels - unsupported_fp8e4m3fnuz_ops.insert("logsoftmax"); - unsupported_fp8e4m3fnuz_ops.insert("nonzero"); - unsupported_fp8e4m3fnuz_ops.insert("prefix_scan_sum"); - unsupported_fp8e4m3fnuz_ops.insert("scatter_none"); - unsupported_fp8e4m3fnuz_ops.insert("topk"); - unsupported_fp8e4m3fnuz_ops.insert("rnn_var_sl_shift_output"); - unsupported_fp8e4m3fnuz_ops.insert("multinomial"); - unsupported_fp8e4m3fnuz_ops.insert("argmax"); - unsupported_fp8e4m3fnuz_ops.insert("argmin"); - - std::set unsupported_fp8e5m2fnuz_ops = unsupported_fp8e4m3fnuz_ops; - // disable gemm for fp8e5m2fnuz if rocBLAS is being used - if(not enabled(MIGRAPHX_ENABLE_HIPBLASLT_GEMM{})) - { - unsupported_fp8e5m2fnuz_ops.insert("dot"); - unsupported_fp8e5m2fnuz_ops.insert("quant_dot"); - } - - std::set unsupported_fp8ocp_ops = {}; - // TODO: remove this when the flag is removed - if(not enabled(MIGRAPHX_ENABLE_HIPBLASLT_GEMM{})) - { - unsupported_fp8ocp_ops.insert("dot"); - unsupported_fp8ocp_ops.insert("quant_dot"); - } -#if MIGRAPHX_USE_MIOPEN - // MIOpen doesn't have support for fp8 pooling yet. - unsupported_fp8ocp_ops.insert("pooling"); -#endif - if(not gpu::gfx_has_fp8ocp_intrinsics()) - { - unsupported_fp8ocp_ops.insert("convolution"); - unsupported_fp8ocp_ops.insert("quant_convolution"); - unsupported_fp8ocp_ops.insert("dot"); - unsupported_fp8ocp_ops.insert("quant_dot"); - } - // add all device kernels - unsupported_fp8ocp_ops.insert("logsoftmax"); - unsupported_fp8ocp_ops.insert("nonzero"); - unsupported_fp8ocp_ops.insert("prefix_scan_sum"); - unsupported_fp8ocp_ops.insert("scatter_none"); - unsupported_fp8ocp_ops.insert("topk"); - unsupported_fp8ocp_ops.insert("rnn_var_sl_shift_output"); - unsupported_fp8ocp_ops.insert("multinomial"); - unsupported_fp8ocp_ops.insert("argmax"); - unsupported_fp8ocp_ops.insert("argmin"); - - // clang-format off - return - { - split_single_dyn_dim{}, - dead_code_elimination{}, - simplify_dyn_ops{}, - dead_code_elimination{}, - normalize_ops{}, - dead_code_elimination{}, - eliminate_identity{}, - dead_code_elimination{}, - enable_pass(not gpu::gfx_has_fp8ocp_intrinsics() and gpu::gfx_has_fp8fnuz_intrinsics(), fp8_ocp_to_fnuz{}), - enable_pass(not gpu::gfx_has_fp8ocp_intrinsics() and gpu::gfx_has_fp8fnuz_intrinsics(), dead_code_elimination{}), - simplify_qdq{}, - enable_pass(not mlir_enabled(), rewrite_quantization{}), - dead_code_elimination{}, - // workaround for rocBLAS unsupported error when using uint8 in quant_dot, quant_convolution & pooling - eliminate_data_type{{migraphx::shape::uint8_type}, shape::float_type, {"quant_convolution", "quant_dot", "pooling"}}, - eliminate_data_type{unsupported_types, shape::type_t::float_type}, - simplify_reshapes{}, - eliminate_identity{}, - eliminate_pad{}, - dead_code_elimination{}, - insert_pad{{"convolution"}}, - dead_code_elimination{}, - rewrite_rnn{}, - dead_code_elimination{}, - inline_module{}, - rewrite_pooling{}, - dead_code_elimination{}, - rewrite_gelu{options.fast_math}, - optimize_module{}, - layout_convolution{.channels_last = enabled(MIGRAPHX_ENABLE_NHWC{})}, - dead_code_elimination{}, - prefuse_ops{}, - dead_code_elimination{}, - eliminate_data_type{{migraphx::shape::fp8e4m3fnuz_type}, shape::float_type, unsupported_fp8e4m3fnuz_ops}, - eliminate_data_type{{migraphx::shape::fp8e5m2fnuz_type}, shape::float_type, unsupported_fp8e5m2fnuz_ops}, - eliminate_data_type{{migraphx::shape::fp8e4m3fn_type, migraphx::shape::fp8e5m2_type}, shape::float_type, unsupported_fp8ocp_ops}, - dead_code_elimination{}, - rewrite_reduce{}, - rewrite_low_precision{}, - dead_code_elimination{}, - optimize_module{}, - fuse_pointwise_reduce{}, - dead_code_elimination{}, -#ifndef _WIN32 - enable_pass(enabled(MIGRAPHX_ENABLE_CK{}), fuse_ck{}), -#endif - dead_code_elimination{}, - enable_pass(mlir_enabled(), fuse_mlir{&ctx}), - dead_code_elimination{}, - fuse_concat{}, - dead_code_elimination{}, - auto_contiguous{}, - dead_code_elimination{}, - lowering{&ctx, options.offload_copy}, - eliminate_contiguous{"gpu::contiguous"}, - dead_code_elimination{}, - eliminate_concat{concat_gpu_optimization{}}, - dead_code_elimination{}, -#if MIGRAPHX_USE_MIOPEN - compile_miopen{&gctx}, - dead_code_elimination{}, -#endif - fuse_ops{&ctx, options.fast_math}, - dead_code_elimination{}, -#if MIGRAPHX_USE_HIPBLASLT - compile_hipblaslt{&gctx}, - dead_code_elimination{}, -#endif - replace_allocate{gpu_allocation_model{}, options.offload_copy}, - dead_code_elimination{}, - adjust_allocation{gpu_allocation_model{}}, - dead_code_elimination{}, - compile_ops{&ctx, options.exhaustive_tune}, - dead_code_elimination{}, - promote_literals{}, - dead_code_elimination{}, - write_literals{&ctx}, - schedule{gpu::schedule_model{ctx.get_current_device().nstreams()}, not enabled(MIGRAPHX_DISABLE_SCHEDULE_PASS{})}, - memory_coloring{"hip::allocate"}, - sync_device{}, - preallocate_param{"scratch", gpu_allocation_model{}}, - dead_code_elimination{}, - eliminate_allocation{"hip::allocate"}, - check_context{}, - normalize_ops{}, - dead_code_elimination{}, - eliminate_identity{} - }; - // clang-format on -} - -std::string target::name() const { return "gpu"; } - -migraphx::context target::get_context() const { return context(gpu::get_device_id()); } - -argument target::copy_to(const argument& arg) const { return gpu::to_gpu(arg); } - -argument target::copy_from(const argument& arg) const { return gpu::from_gpu(arg); } - -argument target::allocate(const shape& s) const { return gpu::allocate_gpu(s); } - -MIGRAPHX_REGISTER_TARGET(target); - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/time_op.cpp b/docker/rocm/migraphx/targets/gpu/time_op.cpp deleted file mode 100644 index 3b37cfb1f..000000000 --- a/docker/rocm/migraphx/targets/gpu/time_op.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -std::vector generate_arguments(const std::vector& shapes, - unsigned long seed = 0, - random_mode rm = random_mode::random) -{ - std::vector args; - std::transform(shapes.begin(), shapes.end(), std::back_inserter(args), [&](const auto& s) { - return to_gpu(generate_argument(s, seed++, rm)); - }); - return args; -} - -template -double time_loop(migraphx::gpu::context& gctx, int n, F f) -{ - auto start = context::create_event_for_timing(); - auto stop = context::create_event_for_timing(); - f(); - gctx.get_stream().record(start.get()); - for(auto i : range(n)) - { - (void)i; - f(); - } - gctx.get_stream().record(stop.get()); - gctx.finish(); - return context::get_elapsed_ms(start.get(), stop.get()) / n; -} - -double time_op(const context& ictx, operation op, const std::vector& inputs, int n) -{ - // TODO: Use std::ref - migraphx::context ctx = ictx; - auto& gctx = any_cast(ctx); - auto output = op.compute_shape(inputs); - op.finalize(ctx, output, inputs); - auto args = generate_arguments(inputs); - auto run = [&] { op.compute(ctx, output, args); }; - return time_loop(gctx, n, run); -} - -double time_op(const context& ictx, operation op, int n) -{ - auto inputs = any_cast(op).expected_inputs; - return time_op(ictx, op, inputs, n); -} - -double time_program(const context& ictx, program p, int n) -{ - std::vector ctx_vec = {ictx}; - auto& gctx = any_cast(ctx_vec.front()); - auto* mm = p.get_main_module(); - mm->finalize(ctx_vec); - auto in_shapes = p.get_parameter_shapes(); - std::unordered_map param_map; - unsigned long seed = 0; - for(const auto& [name, shape] : in_shapes) - { - param_map[name] = to_gpu(generate_argument(shape, seed++, random_mode::random)); - } - auto run = [&] { p.eval_with_context(ctx_vec, param_map); }; - return time_loop(gctx, n, run); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/topk.cpp b/docker/rocm/migraphx/targets/gpu/topk.cpp deleted file mode 100644 index 2e799c650..000000000 --- a/docker/rocm/migraphx/targets/gpu/topk.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -shape hip_topk::compute_shape(std::vector inputs) const -{ - return op.normalize_compute_shape({inputs.front()}); -} - -argument hip_topk::compute(context& ctx, const shape&, const std::vector& args) const -{ - auto outputs = args.back().get_sub_objects(); - return op.largest ? device::topk_largest(ctx.get_stream().get(), - outputs.front(), - outputs.back(), - args[0], - op.k, - op.axis) - : device::topk_smallest(ctx.get_stream().get(), - outputs.front(), - outputs.back(), - args[0], - op.k, - op.axis); -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/gpu/write_literals.cpp b/docker/rocm/migraphx/targets/gpu/write_literals.cpp deleted file mode 100644 index cbc776737..000000000 --- a/docker/rocm/migraphx/targets/gpu/write_literals.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace gpu { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_COPY_LITERALS) - -void write_literals::apply(module& m) const -{ - assert(ctx != nullptr); - std::size_t n = 0; - for(auto ins : iterator_for(m)) - { - if(ins->name() == "@literal") - { - if(enabled(MIGRAPHX_COPY_LITERALS{})) - { - literal l = ins->get_literal(); - auto pre = m.add_literal(l); - auto alloc = m.insert_instruction(std::next(pre), hip_allocate{l.get_shape()}); - m.replace_instruction(ins, hip_copy_to_gpu{}, pre, alloc); - } - else - { - std::string id = m.name() + ":@literal:" + std::to_string(n); - m.replace_instruction(ins, hip_copy_literal{ins->get_literal(), id}); - n++; - } - } - } -} - -} // namespace gpu -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/ref/CMakeLists.txt b/docker/rocm/migraphx/targets/ref/CMakeLists.txt deleted file mode 100644 index d4b3e63c7..000000000 --- a/docker/rocm/migraphx/targets/ref/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### - -add_library(migraphx_ref - target.cpp - lowering.cpp -) -set_target_properties(migraphx_ref PROPERTIES EXPORT_NAME ref) -rocm_set_soversion(migraphx_ref ${MIGRAPHX_SO_VERSION}) - -rocm_clang_tidy_check(migraphx_ref) -target_link_libraries(migraphx_ref PRIVATE Threads::Threads) -target_link_libraries(migraphx_ref PUBLIC migraphx) - -migraphx_generate_export_header(migraphx_ref) - -rocm_install_targets( - PRIVATE - TARGETS migraphx_ref - INCLUDE - ${CMAKE_CURRENT_SOURCE_DIR}/include -) - diff --git a/docker/rocm/migraphx/targets/ref/include/migraphx/ref/context.hpp b/docker/rocm/migraphx/targets/ref/include/migraphx/ref/context.hpp deleted file mode 100644 index 8c2cdfe9d..000000000 --- a/docker/rocm/migraphx/targets/ref/include/migraphx/ref/context.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CONTEXT_HPP -#define MIGRAPHX_GUARD_RTGLIB_CONTEXT_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace ref { - -struct context -{ - void finish() const {} -}; - -} // namespace ref -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/ref/include/migraphx/ref/lowering.hpp b/docker/rocm/migraphx/targets/ref/include/migraphx/ref/lowering.hpp deleted file mode 100644 index a775fed15..000000000 --- a/docker/rocm/migraphx/targets/ref/include/migraphx/ref/lowering.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_RTGLIB_CPU_LOWERING_HPP -#define MIGRAPHX_GUARD_RTGLIB_CPU_LOWERING_HPP - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace ref { - -struct MIGRAPHX_REF_EXPORT lowering -{ - std::string name() const { return "ref::lowering"; } - void apply(module& m) const; -}; - -} // namespace ref -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/ref/include/migraphx/ref/target.hpp b/docker/rocm/migraphx/targets/ref/include/migraphx/ref/target.hpp deleted file mode 100644 index b31b7f9d1..000000000 --- a/docker/rocm/migraphx/targets/ref/include/migraphx/ref/target.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_MIGRAPHLIB_CPU_TARGET_HPP -#define MIGRAPHX_GUARD_MIGRAPHLIB_CPU_TARGET_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -struct pass; -namespace ref { - -struct MIGRAPHX_REF_EXPORT target -{ - std::string name() const; - std::vector get_passes(migraphx::context& ctx, const compile_options&) const; - migraphx::context get_context() const { return context{}; } - - argument copy_to(const argument& arg) const { return arg; } - argument copy_from(const argument& arg) const { return arg; } - argument allocate(const shape& s) const; -}; - -} // namespace ref -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/targets/ref/lowering.cpp b/docker/rocm/migraphx/targets/ref/lowering.cpp deleted file mode 100644 index a0b6b4bd0..000000000 --- a/docker/rocm/migraphx/targets/ref/lowering.cpp +++ /dev/null @@ -1,504 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace ref { - -template -T zero(const T&) -{ - return T(0); -} - -template -typename std::conditional_t{}, std::make_signed, std::enable_if>:: - type - make_signed(T x) -{ - return x; -} - -struct ref_lrn -{ - op::lrn op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "ref::lrn"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - argument compute(context&, shape output_shape, std::vector args) const - { - argument result{output_shape}; - visit_all(result, args[0])([&](auto output, auto input) { - int n_batch = output_shape.lens()[0]; - int channels = output_shape.lens()[1]; - int height = output_shape.lens()[2]; - int width = output_shape.lens()[3]; - float alphaoverarea = op.alpha / float(op.size); - int radius_lower = (op.size - 1) / 2; - int radius_upper = op.size / 2 + 1; - - par_dfor(n_batch, height, width)([&](int b, int h, int w) { - float scale = 0; - dfor(channels)([&](int c) { - auto start = (c - radius_lower) < 0 ? 0 : (c - radius_lower); - auto end = (c + radius_upper) > channels ? channels : (c + radius_upper); - for(auto k = start; k < end; ++k) - { - scale += std::pow(input(b, k, h, w), 2); - } - scale *= alphaoverarea; - scale += op.bias; - scale = std::pow(scale, -op.beta); - output(b, c, h, w) = input(b, c, h, w) * scale; - }); - }); - }); - return result; - } -}; -MIGRAPHX_REGISTER_OP(ref_lrn) - -template -void visit_quantize_impl(V&& v, T&& x, Ts&&... xs) -{ - x.visit([&](auto y) { visit_all(xs...)([&](auto... ys) { v(y, ys...); }); }); -} - -template -auto visit_quantize(T&& x, Ts&&... xs) -{ - return [&](auto v) { - // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70100 - visit_quantize_impl(v, x, xs...); - }; -} - -struct ref_im2col -{ - op::im2col op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - static std::string name() { return "ref::im2col"; } - shape compute_shape(const std::vector& inputs) const - { - return op.normalize_compute_shape(inputs); - } - - argument compute(context&, const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - auto input_shape = args[0].get_shape(); - auto weights_shape = args[1].get_shape(); - visit_all(result, args[0])([&](auto col, auto input) { - const std::size_t& height = input_shape.lens()[2]; - const std::size_t& width = input_shape.lens()[3]; - const std::size_t& channels = weights_shape.lens()[1]; - const std::size_t& kernel_h = weights_shape.lens()[2]; - const std::size_t& kernel_w = weights_shape.lens()[3]; - const std::size_t& pad_h = op.padding[0]; - const std::size_t& pad_w = op.padding[1]; - const std::size_t& stride_h = op.stride[0]; - const std::size_t& stride_w = op.stride[1]; - - long kdiv2_h = long(kernel_h) / 2; - long kdiv2_w = long(kernel_w) / 2; - // calculate output sizes - const std::size_t col_height = (height - kernel_h + 2 * pad_h) / stride_h + 1; - const std::size_t col_width = (width - kernel_w + 2 * pad_w) / stride_w + 1; - // account for padding for the starting position of the input pixels - long iinput = kdiv2_h - long(pad_h); - // loop over output pixels (ioutput, joutput) - for(std::size_t ioutput = 0; ioutput < col_height; ioutput++, iinput += stride_h) - { - long jinput = kdiv2_w - long(pad_w); - for(std::size_t joutput = 0; joutput < col_width; joutput++, jinput += stride_w) - { - // compute linear index for output - std::size_t ldx = ioutput * col_width + joutput; - std::size_t p = 0; - dfor(channels, - kernel_h, - kernel_w)([&](std::size_t c, std::size_t koffset, std::size_t loffset) { - auto idx = iinput + long(koffset) - kdiv2_h; - auto jdx = jinput + long(loffset) - kdiv2_w; - col(ldx, p) = - ((idx >= 0) and (idx < height) and (jdx >= 0) and (jdx < width)) - ? input(0, c, idx, jdx) - : 0; - p++; - }); - } - } - }); - return result; - } -}; -MIGRAPHX_REGISTER_OP(ref_im2col) - -struct ref_op -{ - operation op = op::identity{}; - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - std::string name() const { return "ref::op"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - argument compute(context&, const shape& output_shape, const std::vector& args) const - { - return op.compute(output_shape, args); - } - value to_value() const - { - value v; - v["name"] = op.name(); - v["operator"] = op.to_value(); - return v; - } - void from_value(const value& v) - { - op = make_op(v.at("name").to(), v.at("operator")); - } - friend std::ostream& operator<<(std::ostream& os, const ref_op& x) - { - os << "ref::" << x.op; - return os; - } -}; -MIGRAPHX_REGISTER_OP(ref_op) - -struct ref_pad -{ - op::pad op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "ref::pad"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - argument compute(context&, const dyn_output& dyn_out, std::vector args) const - { - assert(dyn_out.computed_shape.standard()); - argument result{dyn_out.computed_shape}; - result.visit([&](auto output) { - using type = typename decltype(output)::value_type; - std::fill(output.begin(), output.end(), pad_clamp(op.value)); - }); - - visit_all(result, args[0])([&](auto output, auto input) { - shape_for_each(input.get_shape(), [&](const auto& idx) { - std::vector new_idx(idx.size()); - std::transform( - idx.begin(), idx.end(), op.pads.begin(), new_idx.begin(), [](auto i, auto j) { - return i + j; - }); - output(new_idx.begin(), new_idx.end()) = input(idx.begin(), idx.end()); - }); - }); - - return result; - } -}; -MIGRAPHX_REGISTER_OP(ref_pad) - -struct ref_gemm -{ - op::dot op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - std::string name() const { return "ref::dot"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - - argument compute(context&, const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - visit_all(result, args[0], args[1])( - [&](auto cmat, auto amat, auto bmat) { gemm(cmat, amat, bmat, 1.0f, 0.0f); }); - return result; - } -}; -MIGRAPHX_REGISTER_OP(ref_gemm) - -struct ref_quant_gemm -{ - op::quant_dot op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "ref::quant_dot"; } - shape compute_shape(const std::vector& inputs) const { return op.compute_shape(inputs); } - - argument compute(context&, const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - result.visit([&](auto cmat) { - visit_all(args.at(0), args.at(1))( - [&](auto amat, auto bmat) { return gemm(cmat, amat, bmat, 1.0f, 0.0f); }); - }); - return result; - } -}; - -MIGRAPHX_REGISTER_OP(ref_gemm) - -template -struct ref_softmax : auto_register_op> -{ - ref_softmax() = default; - - ref_softmax(Op pop) : op(std::move(pop)) {} - - Op op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "ref::" + op.name(); } - shape compute_shape(const std::vector& inputs) const - { - return op.normalize_compute_shape(inputs); - } - argument compute(context&, const dyn_output& dyn_out, std::vector args) const - { - argument result{dyn_out.computed_shape}; - auto batch_lens = dyn_out.computed_shape.lens(); - int64_t tuned_axis = tune_axis(args[0].get_shape().lens().size(), op.axis, op.name()); - std::size_t n_dims = batch_lens[tuned_axis]; - batch_lens[tuned_axis] = 1; - shape batch_shape{shape::int32_type, batch_lens}; - - visit_all(result, args[0])([&](auto output, auto input) { - using value_type = accumulator_type; - std::vector batch_max(batch_shape.elements(), - std::numeric_limits::lowest()); - std::vector batch_sum(batch_shape.elements(), value_type(0)); - par_for(batch_shape.elements(), [&](auto i) { - auto idx = batch_shape.multi(i); - for(std::size_t j = 0; j < n_dims; ++j) - { - idx[tuned_axis] = j; - batch_max[i] = - std::max(batch_max[i], input(idx.begin(), idx.end())); - } - - for(std::size_t j = 0; j < n_dims; ++j) - { - idx[tuned_axis] = j; - std::size_t index = dyn_out.computed_shape.index(idx); - output[index] = std::exp(input[index] - batch_max[i]); - } - - for(std::size_t j = 0; j < n_dims; ++j) - { - idx[tuned_axis] = j; - batch_sum[i] += output(idx.begin(), idx.end()); - } - - for(std::size_t j = 0; j < n_dims; ++j) - { - idx[tuned_axis] = j; - output(idx.begin(), idx.end()) = - op.output()(output(idx.begin(), idx.end()), batch_sum[i]); - } - }); - }); - - return result; - } -}; - -struct ref_rnn_var_sl_last_output -{ - op::rnn_var_sl_last_output op; - - template - static auto reflect(Self& self, F f) - { - return migraphx::reflect(self.op, f); - } - - std::string name() const { return "ref::rnn_var_sl_last_output"; } - - shape compute_shape(std::vector inputs) const - { - return op.compute_shape(std::move(inputs)); - } - - argument compute(const shape& output_shape, std::vector args) const - { - argument result{output_shape}; - auto out_comp_lens = args[0].get_shape().lens(); - out_comp_lens[0] = 1; - shape out_comp_s{output_shape.type(), out_comp_lens}; - - visit_all(result, args[0])([&](auto output, auto input) { - args[1].visit([&](auto seq_lens) { - par_for(output_shape.elements(), [&](auto i) { - auto idx = out_comp_s.multi(i); - auto b = idx[2]; - if(op.direction == op::rnn_direction::reverse or idx[1] == 1) - { - idx[0] = 0; - } - else - { - idx[0] = seq_lens[b] - 1; - } - output[i] = input(idx.begin(), idx.end()); - }); - }); - }); - - return result; - } -}; -MIGRAPHX_REGISTER_OP(ref_rnn_var_sl_last_output) - -struct ref_apply -{ - module* mod; - std::unordered_map> apply_map{}; - - template - auto simple_op() - { - return [this](instruction_ref ins) { apply_simple_op(ins); }; - } - - template - auto extend_op() - { - return [this](instruction_ref ins) { apply_extend_op(ins); }; - } - - void init() - { - apply_map["dot"] = extend_op(); - apply_map["quant_dot"] = extend_op(); - apply_map["im2col"] = extend_op(); - apply_map["logsoftmax"] = extend_op, op::logsoftmax>(); - apply_map["lrn"] = extend_op(); - apply_map["pad"] = extend_op(); - apply_map["softmax"] = extend_op, op::softmax>(); - apply_map["rnn_var_sl_last_output"] = - extend_op(); - } - - void apply() - { - init(); - for(auto it : iterator_for(*mod)) - { - if(apply_map.count(it->name()) > 0) - { - apply_map.at(it->name())(it); - } - else if(is_context_free(it->get_operator())) - { - apply_ref_op(it); - } - } - } - - void apply_ref_op(instruction_ref ins) const - { - mod->replace_instruction(ins, ref_op{ins->get_operator()}, ins->inputs()); - } - - template - void apply_simple_op(instruction_ref ins) - { - mod->replace_instruction(ins, T{}, ins->inputs()); - } - - template - void apply_extend_op(instruction_ref ins) - { - auto&& op = any_cast(ins->get_operator()); - mod->replace_instruction(ins, T{op}, ins->inputs()); - } -}; - -void lowering::apply(module& m) const { ref_apply{&m}.apply(); } - -} // namespace ref -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/targets/ref/target.cpp b/docker/rocm/migraphx/targets/ref/target.cpp deleted file mode 100644 index 13c15e541..000000000 --- a/docker/rocm/migraphx/targets/ref/target.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace ref { - -std::string target::name() const { return "ref"; } - -std::vector target::get_passes(migraphx::context&, const compile_options&) const -{ - return {normalize_ops{}, - eliminate_pad{}, - dead_code_elimination{}, - insert_pad{}, - dead_code_elimination{}, - rewrite_rnn{}, - dead_code_elimination{}, - auto_contiguous{}, - dead_code_elimination{}, - lowering{}, - dead_code_elimination{}}; -} - -argument target::allocate(const shape& s) const { return fill_argument(s, 0); } - -MIGRAPHX_REGISTER_TARGET(target); - -} // namespace ref -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/CMakeLists.txt b/docker/rocm/migraphx/tf/CMakeLists.txt deleted file mode 100644 index 49df6d39d..000000000 --- a/docker/rocm/migraphx/tf/CMakeLists.txt +++ /dev/null @@ -1,66 +0,0 @@ -##################################################################################### -# The MIT License (MIT) -# -# Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -##################################################################################### -find_package(Protobuf REQUIRED) - -protobuf_generate_cpp( - PROTO_SRCS PROTO_HDRS - graph.proto - node_def.proto - attr_value.proto - tensor.proto - tensor_shape.proto - resource_handle.proto - types.proto - function.proto - op_def.proto - versions.proto -) -add_library(tf-proto STATIC ${PROTO_SRCS}) -target_include_directories(tf-proto SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${PROTOBUF_INCLUDE_DIR}) -if(MSVC) - target_compile_options(tf-proto PRIVATE /w) -else() - target_compile_options(tf-proto PRIVATE -w) -endif() -target_link_libraries(tf-proto PRIVATE ${PROTOBUF_LIBRARY}) -set_target_properties(tf-proto PROPERTIES POSITION_INDEPENDENT_CODE On) - -file(GLOB TF_SRCS CONFIGURE_DEPENDS *.cpp) -add_library(migraphx_tf ${TF_SRCS}) -migraphx_generate_export_header(migraphx_tf) -target_include_directories(migraphx_tf PRIVATE include) -set_target_properties(migraphx_tf PROPERTIES EXPORT_NAME tf) -rocm_set_soversion(migraphx_tf ${MIGRAPHX_SO_VERSION}) -rocm_clang_tidy_check(migraphx_tf) -target_link_libraries(migraphx_tf PRIVATE tf-proto) -if(NOT WIN32) - target_link_libraries(migraphx_tf PRIVATE "-Wl,--exclude-libs,ALL") -endif() -target_link_libraries(migraphx_tf PUBLIC migraphx) - -rocm_install_targets( - PRIVATE - TARGETS migraphx_tf -) - diff --git a/docker/rocm/migraphx/tf/attr_value.proto b/docker/rocm/migraphx/tf/attr_value.proto deleted file mode 100644 index 76944f77b..000000000 --- a/docker/rocm/migraphx/tf/attr_value.proto +++ /dev/null @@ -1,62 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "AttrValueProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensor.proto"; -import "tensor_shape.proto"; -import "types.proto"; - -// Protocol buffer representing the value for an attr used to configure an Op. -// Comment indicates the corresponding attr type. Only the field matching the -// attr type may be filled. -message AttrValue { - // LINT.IfChange - message ListValue { - repeated bytes s = 2; // "list(string)" - repeated int64 i = 3 [packed = true]; // "list(int)" - repeated float f = 4 [packed = true]; // "list(float)" - repeated bool b = 5 [packed = true]; // "list(bool)" - repeated DataType type = 6 [packed = true]; // "list(type)" - repeated TensorShapeProto shape = 7; // "list(shape)" - repeated TensorProto tensor = 8; // "list(tensor)" - repeated NameAttrList func = 9; // "list(attr)" - } - // LINT.ThenChange(https://www.tensorflow.org/code/tensorflow/c/c_api.cc) - - oneof value { - bytes s = 2; // "string" - int64 i = 3; // "int" - float f = 4; // "float" - bool b = 5; // "bool" - DataType type = 6; // "type" - TensorShapeProto shape = 7; // "shape" - TensorProto tensor = 8; // "tensor" - ListValue list = 1; // any "list(...)" - - // "func" represents a function. func.name is a function's name or - // a primitive op's name. func.attr.first is the name of an attr - // defined for that function. func.attr.second is the value for - // that attr in the instantiation. - NameAttrList func = 10; - - // This is a placeholder only used in nodes defined inside a - // function. It indicates the attr value will be supplied when - // the function is instantiated. For example, let us suppose a - // node "N" in function "FN". "N" has an attr "A" with value - // placeholder = "foo". When FN is instantiated with attr "foo" - // set to "bar", the instantiated node N's attr A will have been - // given the value "bar". - string placeholder = 9; - } -} - -// A list of attr names and their values. The whole list is attached -// with a string name. E.g., MatMul[T=float]. -message NameAttrList { - string name = 1; - map attr = 2; -} diff --git a/docker/rocm/migraphx/tf/function.proto b/docker/rocm/migraphx/tf/function.proto deleted file mode 100644 index ce7f8d60e..000000000 --- a/docker/rocm/migraphx/tf/function.proto +++ /dev/null @@ -1,102 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "FunctionProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "attr_value.proto"; -import "node_def.proto"; -import "op_def.proto"; - -// A library is a set of named functions. -message FunctionDefLibrary { - repeated FunctionDef function = 1; - repeated GradientDef gradient = 2; -} - -// A function can be instantiated when the runtime can bind every attr -// with a value. When a GraphDef has a call to a function, it must -// have binding for every attr defined in the signature. -// -// TODO(zhifengc): -// * device spec, etc. -message FunctionDef { - // The definition of the function's name, arguments, return values, - // attrs etc. - OpDef signature = 1; - - // Attributes specific to this function definition. - map attr = 5; - - // NOTE: field id 2 deleted on Jan 11, 2017, GraphDef version 21. - reserved 2; - - // In both of the following fields, there is the need to specify an - // output that is used as either the input to another node (in - // `node_def`) or as a return value of the function (in `ret`). - // Unlike the NodeDefs in GraphDef, we need to be able to specify a - // list in some cases (instead of just single outputs). Also, we - // need to be able to deal with lists of unknown length (so the - // output index may not be known at function definition time). So - // we use the following format instead: - // * "fun_in" where "fun_in" is the name of a function input arg in - // the `signature` field above. This represents that input, whether - // it is a single tensor or a list. - // * "fun_in:0" gives the first element of a function input arg (a - // non-list input is considered a list of length 1 for these - // purposes). - // * "node:out" where "node" is the name of a node in `node_def` and - // "out" is the name one of its op's output arguments (the name - // comes from the OpDef of the node's op). This represents that - // node's output, whether it is a single tensor or a list. - // Note: We enforce that an op's output arguments are never - // renamed in the backwards-compatibility test. - // * "node:out:0" gives the first element of a node output arg (a - // non-list output is considered a list of length 1 for these - // purposes). - // - // NOT CURRENTLY SUPPORTED (but may be in the future): - // * "node:out:-1" gives last element in a node output list - // * "node:out:1:" gives a list with all but the first element in a - // node output list - // * "node:out::-1" gives a list with all but the last element in a - // node output list - - // The body of the function. Unlike the NodeDefs in a GraphDef, attrs - // may have values of type `placeholder` and the `input` field uses - // the "output" format above. - - // By convention, "op" in node_def is resolved by consulting with a - // user-defined library first. If not resolved, "func" is assumed to - // be a builtin op. - repeated NodeDef node_def = 3; - - // A mapping from the output arg names from `signature` to the - // outputs from `node_def` that should be returned by the function. - map ret = 4; -} - -// GradientDef defines the gradient function of a function defined in -// a function library. -// -// A gradient function g (specified by gradient_func) for a function f -// (specified by function_name) must follow the following: -// -// The function 'f' must be a numerical function which takes N inputs -// and produces M outputs. Its gradient function 'g', which is a -// function taking N + M inputs and produces N outputs. -// -// I.e. if we have -// (y1, y2, ..., y_M) = f(x1, x2, ..., x_N), -// then, g is -// (dL/dx1, dL/dx2, ..., dL/dx_N) = g(x1, x2, ..., x_N, -// dL/dy1, dL/dy2, ..., dL/dy_M), -// where L is a scalar-value function of (x1, x2, ..., xN) (e.g., the -// loss function). dL/dx_i is the partial derivative of L with respect -// to x_i. -message GradientDef { - string function_name = 1; // The function name. - string gradient_func = 2; // The gradient function's name. -} diff --git a/docker/rocm/migraphx/tf/graph.proto b/docker/rocm/migraphx/tf/graph.proto deleted file mode 100644 index 14d9edfab..000000000 --- a/docker/rocm/migraphx/tf/graph.proto +++ /dev/null @@ -1,56 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "GraphProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "node_def.proto"; -import "function.proto"; -import "versions.proto"; - -// Represents the graph of operations -message GraphDef { - repeated NodeDef node = 1; - - // Compatibility versions of the graph. See core/public/version.h for version - // history. The GraphDef version is distinct from the TensorFlow version, and - // each release of TensorFlow will support a range of GraphDef versions. - VersionDef versions = 4; - - // Deprecated single version field; use versions above instead. Since all - // GraphDef changes before "versions" was introduced were forward - // compatible, this field is entirely ignored. - int32 version = 3 [deprecated = true]; - - // EXPERIMENTAL. DO NOT USE OR DEPEND ON THIS YET. - // - // "library" provides user-defined functions. - // - // Naming: - // * library.function.name are in a flat namespace. - // NOTE: We may need to change it to be hierarchical to support - // different orgs. E.g., - // { "/google/nn", { ... }}, - // { "/google/vision", { ... }} - // { "/org_foo/module_bar", { ... }} - // map named_lib; - // * If node[i].op is the name of one function in "library", - // node[i] is deemed as a function call. Otherwise, node[i].op - // must be a primitive operation supported by the runtime. - // - // - // Function call semantics: - // - // * The callee may start execution as soon as some of its inputs - // are ready. The caller may want to use Tuple() mechanism to - // ensure all inputs are ready in the same time. - // - // * The consumer of return values may start executing as soon as - // the return values the consumer depends on are ready. The - // consumer may want to use Tuple() mechanism to ensure the - // consumer does not start until all return values of the callee - // function are ready. - FunctionDefLibrary library = 2; -}; diff --git a/docker/rocm/migraphx/tf/include/migraphx/tf/op_parser.hpp b/docker/rocm/migraphx/tf/include/migraphx/tf/op_parser.hpp deleted file mode 100644 index 7ac7af501..000000000 --- a/docker/rocm/migraphx/tf/include/migraphx/tf/op_parser.hpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_TF_REGISTER_OP_PARSER_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_TF_REGISTER_OP_PARSER_HPP - -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct op_desc -{ - std::string tf_name = ""; - std::string op_name = ""; -}; - -void register_op_parser(const std::string& name, tf_parser::op_func f); -tf_parser::op_func get_op_parser(const std::string& name); -std::vector get_op_parsers(); - -inline std::vector implicit_multi_op(std::vector inss) -{ - return inss; -} - -inline std::vector implicit_multi_op(instruction_ref ins) { return {ins}; } - -template -void register_op_parser() -{ - T parser; - for(auto&& opd : parser.operators()) - register_op_parser(opd.tf_name, - [opd, parser](auto&&... xs) { return parser.base_parse(opd, xs...); }); -} - -struct register_op_parser_action -{ - template - static void apply() - { - register_op_parser(); - } -}; - -template -struct op_parser : auto_register -{ - bool transpose() const { return false; } - std::vector base_parse(const op_desc& opd, - const tf_parser& parser, - tf_parser::node_info info, - const std::vector& args) const - { - std::vector result; - auto& self = static_cast(*this); - if(self.transpose()) - { - result = implicit_multi_op(self.parse(opd, parser, info, parser.to_nchw(args))); - std::transform(result.begin(), result.end(), result.begin(), [&](auto ins) { - return parser.to_nhwc(ins); - }); - } - else - { - result = implicit_multi_op(self.parse(opd, parser, info, args)); - } - return result; - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/tf/include/migraphx/tf/tf_parser.hpp b/docker/rocm/migraphx/tf/include/migraphx/tf/tf_parser.hpp deleted file mode 100644 index 99510512e..000000000 --- a/docker/rocm/migraphx/tf/include/migraphx/tf/tf_parser.hpp +++ /dev/null @@ -1,141 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#ifndef MIGRAPHX_GUARD_AMDMIGRAPHX_TF_PARSER_HPP -#define MIGRAPHX_GUARD_AMDMIGRAPHX_TF_PARSER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -// namespace tf = tf_for_migraphx; - -struct tf_parser -{ - std::string filename; - std::string path = "."; - using attribute_map = std::unordered_map; - struct node_info - { - attribute_map attributes{}; - std::string name = ""; - module* mm = nullptr; - - instruction_ref make_contiguous(instruction_ref ins) const; - - instruction_ref add_broadcastable_binary_op(const std::string& op_name, - instruction_ref arg0, - instruction_ref arg1) const; - - instruction_ref add_common_op(const std::string& op_name, - std::vector inputs) const; - - template - instruction_ref add_common_op(const std::string& op_name, Ts... xs) const - { - return add_common_op(op_name, {xs...}); - } - - instruction_ref add_instruction(const operation& op, - const std::vector& args) const; - - template - instruction_ref add_instruction(const operation& op, Ts... xs) const - { - return add_instruction(op, {xs...}); - } - instruction_ref add_literal(literal l) const; - template - instruction_ref add_literal(Ts&&... xs) const - { - return add_literal(literal{std::forward(xs)...}); - } - }; - - using node_map = std::map; - using op_func = std::function( - const tf_parser&, const node_info&, std::vector)>; - node_map nodes; - std::vector input_nodes; - std::vector output_node_names; - std::unordered_map instructions; - program prog = program(); - module* mm = prog.get_main_module(); - bool is_nhwc = true; - unsigned int batch_size = 1; - std::size_t default_dim_value = 1; - std::unordered_map> map_input_dims; - - std::unordered_map ops; - - tf_parser(); - operation load(const std::string& name, const node_info& info) const; - bool should_transpose(instruction_ref ins) const; - instruction_ref to_nhwc(instruction_ref ins) const; - instruction_ref to_nchw(instruction_ref ins) const; - instruction_ref to_kcxy(instruction_ref ins) const; - std::vector to_nchw(const std::vector& args) const; - std::vector to_nhwc(const std::vector& args) const; - int64_t parse_axis(int64_t dim, size_t num_dims) const; - // tf stores certain attributes such as strides, dilations, as a 4D input. - // The first and last dims are equal to 1, and the relevant data is in dims 2 and 3. - // This helper function reorders the data to store for the respective operator member variables. - template - void reorder_data(std::vector& prev_data) const - { - std::vector new_data(prev_data.size()); - for(size_t i = 0; i < new_data.size(); i++) - { - auto new_idx = parse_axis(i, new_data.size()); - new_data.at(new_idx) = prev_data.at(i); - } - prev_data = new_data; - } - - void parse_undefined(module* mm, const std::string& name); - void parse_from(std::istream& is); - void parse_from(const void* data, std::size_t size); - void parse_graph(const tensorflow::GraphDef& graph); - void parse_node(const std::string& name); - literal parse_tensor(const tensorflow::TensorProto& t) const; - shape::type_t parse_type(tensorflow::DataType t) const; - std::vector find_outputs() const; -}; - -std::vector get_axes_from_mask(size_t num_axes, uint32_t mask); - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx - -#endif diff --git a/docker/rocm/migraphx/tf/node_def.proto b/docker/rocm/migraphx/tf/node_def.proto deleted file mode 100644 index a79c0acd7..000000000 --- a/docker/rocm/migraphx/tf/node_def.proto +++ /dev/null @@ -1,63 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "NodeProto"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "attr_value.proto"; - -message NodeDef { - // The name given to this operator. Used for naming inputs, - // logging, visualization, etc. Unique within a single GraphDef. - // Must match the regexp "[A-Za-z0-9.][A-Za-z0-9_./]*". - string name = 1; - - // The operation name. There may be custom parameters in attrs. - // Op names starting with an underscore are reserved for internal use. - string op = 2; - - // Each input is "node:src_output" with "node" being a string name and - // "src_output" indicating which output tensor to use from "node". If - // "src_output" is 0 the ":0" suffix can be omitted. Regular inputs - // may optionally be followed by control inputs that have the format - // "^node". - repeated string input = 3; - - // A (possibly partial) specification for the device on which this - // node should be placed. - // The expected syntax for this string is as follows: - // - // DEVICE_SPEC ::= PARTIAL_SPEC - // - // PARTIAL_SPEC ::= ("/" CONSTRAINT) * - // CONSTRAINT ::= ("job:" JOB_NAME) - // | ("replica:" [1-9][0-9]*) - // | ("task:" [1-9][0-9]*) - // | ("device:" [A-Za-z]* ":" ([1-9][0-9]* | "*") ) - // - // Valid values for this string include: - // * "/job:worker/replica:0/task:1/device:GPU:3" (full specification) - // * "/job:worker/device:GPU:3" (partial specification) - // * "" (no specification) - // - // If the constraints do not resolve to a single device (or if this - // field is empty or not present), the runtime will attempt to - // choose a device automatically. - string device = 4; - - // Operation-specific graph-construction-time configuration. - // Note that this should include all attrs defined in the - // corresponding OpDef, including those with a value matching - // the default -- this allows the default to change and makes - // NodeDefs easier to interpret on their own. However, if - // an attr with a default is not specified in this list, the - // default will be used. - // The "names" (keys) must match the regexp "[a-z][a-z0-9_]+" (and - // one of the names from the corresponding OpDef's attr field). - // The values must have a type matching the corresponding OpDef - // attr's type field. - // TODO(josh11b): Add some examples here showing best practices. - map attr = 5; -}; diff --git a/docker/rocm/migraphx/tf/op_def.proto b/docker/rocm/migraphx/tf/op_def.proto deleted file mode 100644 index 86bea899a..000000000 --- a/docker/rocm/migraphx/tf/op_def.proto +++ /dev/null @@ -1,166 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "OpDefProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "attr_value.proto"; -import "types.proto"; - -// Defines an operation. A NodeDef in a GraphDef specifies an Op by -// using the "op" field which should match the name of a OpDef. -// LINT.IfChange -message OpDef { - // Op names starting with an underscore are reserved for internal use. - // Names should be CamelCase and match the regexp "[A-Z][a-zA-Z0-9_]*". - string name = 1; - - // For describing inputs and outputs. - message ArgDef { - // Name for the input/output. Should match the regexp "[a-z][a-z0-9_]*". - string name = 1; - - // Human readable description. - string description = 2; - - // Describes the type of one or more tensors that are accepted/produced - // by this input/output arg. The only legal combinations are: - // * For a single tensor: either the "type" field is set or the - // "type_attr" field is set to the name of an attr with type "type". - // * For a sequence of tensors with the same type: the "number_attr" - // field will be set to the name of an attr with type "int", and - // either the "type" or "type_attr" field will be set as for - // single tensors. - // * For a sequence of tensors, the "type_list_attr" field will be set - // to the name of an attr with type "list(type)". - DataType type = 3; - string type_attr = 4; // if specified, attr must have type "type" - string number_attr = 5; // if specified, attr must have type "int" - // If specified, attr must have type "list(type)", and none of - // type, type_attr, and number_attr may be specified. - string type_list_attr = 6; - - // For inputs: if true, the inputs are required to be refs. - // By default, inputs can be either refs or non-refs. - // For outputs: if true, outputs are refs, otherwise they are not. - bool is_ref = 16; - }; - - // Description of the input(s). - repeated ArgDef input_arg = 2; - - // Description of the output(s). - repeated ArgDef output_arg = 3; - - // Description of the graph-construction-time configuration of this - // Op. That is to say, this describes the attr fields that will - // be specified in the NodeDef. - message AttrDef { - // A descriptive name for the argument. May be used, e.g. by the - // Python client, as a keyword argument name, and so should match - // the regexp "[a-z][a-z0-9_]+". - string name = 1; - - // One of the type names from attr_value.proto ("string", "list(string)", - // "int", etc.). - string type = 2; - - // A reasonable default for this attribute if the user does not supply - // a value. If not specified, the user must supply a value. - AttrValue default_value = 3; - - // Human-readable description. - string description = 4; - - // TODO(josh11b): bool is_optional? - - // --- Constraints --- - // These constraints are only in effect if specified. Default is no - // constraints. - - // For type == "int", this is a minimum value. For "list(___)" - // types, this is the minimum length. - bool has_minimum = 5; - int64 minimum = 6; - - // The set of allowed values. Has type that is the "list" version - // of the "type" field above (uses the "list" field of AttrValue). - // If type == "type" or "list(type)" above, then the "type" field - // of "allowed_values.list" has the set of allowed DataTypes. - // If type == "string" or "list(string)", then the "s" field of - // "allowed_values.list" has the set of allowed strings. - AttrValue allowed_values = 7; - } - repeated AttrDef attr = 4; - - // Optional deprecation based on GraphDef versions. - OpDeprecation deprecation = 8; - - // One-line human-readable description of what the Op does. - string summary = 5; - - // Additional, longer human-readable description of what the Op does. - string description = 6; - - // ------------------------------------------------------------------------- - // Which optimizations this operation can participate in. - - // True if the operation is commutative ("op(a,b) == op(b,a)" for all inputs) - bool is_commutative = 18; - - // If is_aggregate is true, then this operation accepts N >= 2 - // inputs and produces 1 output all of the same type. Should be - // associative and commutative, and produce output with the same - // shape as the input. The optimizer may replace an aggregate op - // taking input from multiple devices with a tree of aggregate ops - // that aggregate locally within each device (and possibly within - // groups of nearby devices) before communicating. - // TODO(josh11b): Implement that optimization. - bool is_aggregate = 16; // for things like add - - // Other optimizations go here, like - // can_alias_input, rewrite_when_output_unused, partitioning_strategy, etc. - - // ------------------------------------------------------------------------- - // Optimization constraints. - - // Ops are marked as stateful if their behavior depends on some state beyond - // their input tensors (e.g. variable reading op) or if they have - // a side-effect (e.g. printing or asserting ops). Equivalently, stateless ops - // must always produce the same output for the same input and have - // no side-effects. - // - // By default Ops may be moved between devices. Stateful ops should - // either not be moved, or should only be moved if that state can also - // be moved (e.g. via some sort of save / restore). - // Stateful ops are guaranteed to never be optimized away by Common - // Subexpression Elimination (CSE). - bool is_stateful = 17; // for things like variables, queue - - // ------------------------------------------------------------------------- - // Non-standard options. - - // By default, all inputs to an Op must be initialized Tensors. Ops - // that may initialize tensors for the first time should set this - // field to true, to allow the Op to take an uninitialized Tensor as - // input. - bool allows_uninitialized_input = 19; // for Assign, etc. -}; -// LINT.ThenChange( -// https://www.tensorflow.org/code/tensorflow/core/framework/op_def_util.cc) - -// Information about version-dependent deprecation of an op -message OpDeprecation { - // First GraphDef version at which the op is disallowed. - int32 version = 1; - - // Explanation of why it was deprecated and what to use instead. - string explanation = 2; -}; - -// A collection of OpDefs -message OpList { - repeated OpDef op = 1; -}; diff --git a/docker/rocm/migraphx/tf/op_parser.cpp b/docker/rocm/migraphx/tf/op_parser.cpp deleted file mode 100644 index 9a1b25c9b..000000000 --- a/docker/rocm/migraphx/tf/op_parser.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -std::unordered_map& op_parser_map() -{ - static std::unordered_map m; // NOLINT - return m; -} - -void register_op_parser(const std::string& name, tf_parser::op_func f) -{ - op_parser_map()[name] = std::move(f); -} -tf_parser::op_func get_op_parser(const std::string& name) { return op_parser_map().at(name); } -std::vector get_op_parsers() -{ - std::vector result; - std::transform(op_parser_map().begin(), - op_parser_map().end(), - std::back_inserter(result), - [&](auto&& p) { return p.first; }); - std::sort(result.begin(), result.end()); - return result; -} - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_arg_op.cpp b/docker/rocm/migraphx/tf/parse_arg_op.cpp deleted file mode 100644 index 403382325..000000000 --- a/docker/rocm/migraphx/tf/parse_arg_op.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_arg_op : op_parser -{ - std::vector operators() const { return {{"ArgMax", "argmax"}, {"ArgMin", "argmin"}}; } - - instruction_ref parse(const op_desc& opd, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - const std::vector& args) const - { - int64_t axis = 0; - axis = args[1]->eval().at(); - auto ins = info.add_instruction(make_op(opd.op_name, {{"axis", axis}}), args.front()); - return info.add_instruction(make_op("squeeze", {{"axes", {axis}}}), ins); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_batchnorm.cpp b/docker/rocm/migraphx/tf/parse_batchnorm.cpp deleted file mode 100644 index 4c7772eca..000000000 --- a/docker/rocm/migraphx/tf/parse_batchnorm.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_batchnorm : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"FusedBatchNorm"}, {"FusedBatchNormV3"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - // different default epsilon than from ONNX - float epsilon = 1e-4f; - if(contains(info.attributes, "epsilon")) - { - epsilon = info.attributes.at("epsilon").f(); - } - - auto x_lens = args[0]->get_shape().lens(); - auto x_type = args[0]->get_shape().type(); - - // unsqueeze tensors of shape (C) to broadcast correctly - auto eps = info.add_literal(migraphx::literal{migraphx::shape{x_type}, {epsilon}}); - - auto scale_unsqueeze = - info.add_instruction(migraphx::make_op("unsqueeze", {{"axes", {1, 2}}}), args[1]); - auto bias_unsqueeze = - info.add_instruction(migraphx::make_op("unsqueeze", {{"axes", {1, 2}}}), args[2]); - auto mean_unsqueeze = - info.add_instruction(migraphx::make_op("unsqueeze", {{"axes", {1, 2}}}), args[3]); - auto var_unsqueeze = - info.add_instruction(migraphx::make_op("unsqueeze", {{"axes", {1, 2}}}), args[4]); - - auto x_sub_mean = info.add_broadcastable_binary_op("sub", args[0], mean_unsqueeze); - auto var_eps = info.add_broadcastable_binary_op("add", var_unsqueeze, eps); - auto rsqrt = info.add_instruction(make_op("rsqrt"), var_eps); - auto mul0 = info.add_broadcastable_binary_op("mul", scale_unsqueeze, rsqrt); - auto r0 = info.add_broadcastable_binary_op("mul", x_sub_mean, mul0); - return info.add_broadcastable_binary_op("add", r0, bias_unsqueeze); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_biasadd.cpp b/docker/rocm/migraphx/tf/parse_biasadd.cpp deleted file mode 100644 index 3ecdf42cc..000000000 --- a/docker/rocm/migraphx/tf/parse_biasadd.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_biasadd : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"BiasAdd"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - uint64_t axis = 1; // assume output of previous layer is in NCHW (broadcast on channel) - - auto l0 = info.add_instruction( - make_op("broadcast", {{"axis", axis}, {"out_lens", args[0]->get_shape().lens()}}), - args[1]); - return info.add_instruction(make_op("add"), args[0], l0); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_binary_op.cpp b/docker/rocm/migraphx/tf/parse_binary_op.cpp deleted file mode 100644 index 0aa30f765..000000000 --- a/docker/rocm/migraphx/tf/parse_binary_op.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_binary_op : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const - { - return {{"Add", "add"}, - {"AddV2", "add"}, - {"Mul", "mul"}, - {"Pow", "pow"}, - {"SquaredDifference", "sqdiff"}, - {"Sub", "sub"}}; - } - - instruction_ref parse(const op_desc& opd, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - if(args.size() != 2) - MIGRAPHX_THROW("binary operators should have 2 operands"); - return info.add_broadcastable_binary_op(opd.op_name, args[0], args[1]); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_cast.cpp b/docker/rocm/migraphx/tf/parse_cast.cpp deleted file mode 100644 index 4bcd2905f..000000000 --- a/docker/rocm/migraphx/tf/parse_cast.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_cast : op_parser -{ - std::vector operators() const { return {{"Cast"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& parser, - tf_parser::node_info info, - const std::vector& args) const - { - shape::type_t type = parser.parse_type(info.attributes.at("DstT").type()); - return info.add_instruction(make_op("convert", {{"target_type", type}}), args); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_concat.cpp b/docker/rocm/migraphx/tf/parse_concat.cpp deleted file mode 100644 index 5b0fbb124..000000000 --- a/docker/rocm/migraphx/tf/parse_concat.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_concat : op_parser -{ - std::vector operators() const { return {{"ConcatV2"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - // get index for axis within args - size_t axis_idx = info.attributes.at("N").i(); - int64_t axis = args[axis_idx]->eval().at(); - auto op = make_op("concat", {{"axis", axis}}); - // return only first N arguments (assuming last index is the axis value) - return info.add_instruction( - op, std::vector(args.begin(), args.begin() + args.size() - 1)); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_constant.cpp b/docker/rocm/migraphx/tf/parse_constant.cpp deleted file mode 100644 index 5b1400b15..000000000 --- a/docker/rocm/migraphx/tf/parse_constant.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_constant_op : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"Const"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& parser, - tf_parser::node_info info, - const std::vector& /*args*/) const - { - literal v = parser.parse_tensor(info.attributes.at("value").tensor()); - return info.add_literal(v); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_conv.cpp b/docker/rocm/migraphx/tf/parse_conv.cpp deleted file mode 100644 index cd7b2302c..000000000 --- a/docker/rocm/migraphx/tf/parse_conv.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_conv : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"Conv2D"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& parser, - tf_parser::node_info info, - std::vector args) const - { - op::convolution op; - if(contains(info.attributes, "strides")) - { - std::vector stride; - copy(info.attributes.at("strides").list().i(), std::back_inserter(stride)); - parser.reorder_data(stride); - if(stride.size() != 4) - { - MIGRAPHX_THROW("strides should have 4 values"); - } - op.stride[0] = stride[2]; - op.stride[1] = stride[3]; - } - if(contains(info.attributes, "dilations")) - { - std::vector dilation; - copy(info.attributes.at("dilations").list().i(), std::back_inserter(dilation)); - parser.reorder_data(dilation); - if(dilation.size() != 4) - { - MIGRAPHX_THROW("dilation should have 4 values"); - } - op.dilation[0] = dilation[2]; - op.dilation[1] = dilation[3]; - } - - auto weights = parser.to_kcxy(args[1]); - auto l0 = args[0]; - if(contains(info.attributes, "padding")) - { - const std::string& pad_mode = info.attributes.at("padding").s(); - if(pad_mode.find("SAME") != std::string::npos) - { - std::vector weight_dims = weights->get_shape().lens(); - size_t weight_h = weight_dims[2]; - size_t weight_w = weight_dims[3]; - - auto input_dims = l0->get_shape().lens(); - std::vector pads(input_dims.size()); - calculate_padding(0, pads, input_dims[2], op.stride[0], op.dilation[0], weight_h); - calculate_padding(1, pads, input_dims[3], op.stride[1], op.dilation[1], weight_w); - - op.padding = std::vector(pads.begin(), pads.end()); - } - else if(pad_mode.find("EXPLICIT") != std::string::npos) - { - std::vector padding; - copy(info.attributes.at("explicit_paddings").list().i(), - std::back_inserter(padding)); - if(padding.size() != 4) - { - MIGRAPHX_THROW("padding should have 4 values"); - } - if(padding[0] != padding[2] or padding[1] != padding[3]) - { - MIGRAPHX_THROW("migraphx does not support asymetric padding"); - } - op.padding[0] = padding[0]; - op.padding[1] = padding[1]; - } - } - return info.add_instruction(op, {l0, weights}); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_depthwiseconv.cpp b/docker/rocm/migraphx/tf/parse_depthwiseconv.cpp deleted file mode 100644 index 7474654ad..000000000 --- a/docker/rocm/migraphx/tf/parse_depthwiseconv.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_depthwiseconv : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"DepthwiseConv2dNative"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& parser, - tf_parser::node_info info, - std::vector args) const - { - op::convolution op; - size_t num_channels = args[0]->get_shape().lens()[1]; - op.group = num_channels; - - if(contains(info.attributes, "strides")) - { - std::vector stride; - copy(info.attributes.at("strides").list().i(), std::back_inserter(stride)); - parser.reorder_data(stride); - if(stride.size() != 4) - { - MIGRAPHX_THROW("strides should have 4 values"); - } - op.stride[0] = stride[2]; - op.stride[1] = stride[3]; - } - - auto weights = parser.to_kcxy(args[1]); - if(contains(info.attributes, "dilations")) - { - std::vector dilation; - copy(info.attributes.at("dilations").list().i(), std::back_inserter(dilation)); - parser.reorder_data(dilation); - if(dilation.size() != 4) - { - MIGRAPHX_THROW("dilation should have 4 values"); - } - op.dilation[0] = dilation[2]; - op.dilation[1] = dilation[3]; - } - - auto l0 = args[0]; - if(contains(info.attributes, "padding")) - { - const std::string& pad_mode = info.attributes.at("padding").s(); - - if(pad_mode.find("SAME") != std::string::npos) - { - std::vector weight_dims = weights->get_shape().lens(); - size_t weight_h = weight_dims[2]; - size_t weight_w = weight_dims[3]; - - auto input_dims = l0->get_shape().lens(); - std::vector pads(input_dims.size()); - calculate_padding(0, pads, input_dims[2], op.stride[0], op.dilation[0], weight_h); - calculate_padding(1, pads, input_dims[3], op.stride[1], op.dilation[1], weight_w); - - if(pads[0] != pads[2] or pads[1] != pads[3]) - { - std::vector padding = {0, 0, pads[0], pads[1], 0, 0, pads[2], pads[3]}; - l0 = info.add_instruction(migraphx::make_op("pad", {{"pads", padding}}), l0); - } - else - { - op.padding[0] = pads[0]; - op.padding[1] = pads[1]; - } - } - } - - std::vector new_weights_shape; - copy(weights->get_shape().lens(), std::back_inserter(new_weights_shape)); - - // weight format is (out_channels, in_channels, h, w), but in depthwise_conv, - // out_channels is equal to the multiplier. Adjust by inserting a reshape and - // setting in_channels to 1 - int64_t multiplier = new_weights_shape[0]; - int64_t out_channels = num_channels * multiplier; - new_weights_shape[0] = out_channels; - new_weights_shape[1] = 1; - // Make sure weights are contiguous before doing reshape - auto new_weights = info.add_instruction(make_op("reshape", {{"dims", new_weights_shape}}), - info.make_contiguous(weights)); - - return info.add_instruction(op, {l0, new_weights}); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_expanddims.cpp b/docker/rocm/migraphx/tf/parse_expanddims.cpp deleted file mode 100644 index db74eb9f6..000000000 --- a/docker/rocm/migraphx/tf/parse_expanddims.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_expanddims : op_parser -{ - std::vector operators() const { return {{"ExpandDims"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - std::vector input_dims = args[0]->get_shape().lens(); - std::vector new_dims(input_dims.begin(), input_dims.end()); - size_t num_dims = input_dims.size(); - int32_t dim = args[1]->eval().at(); - - if(dim < 0) - { - new_dims.insert(new_dims.begin() + (num_dims + dim + 1), 1); - } - else - { - new_dims.insert(new_dims.begin() + dim, 1); - } - return info.add_instruction(make_op("reshape", {{"dims", new_dims}}), args[0]); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_gather.cpp b/docker/rocm/migraphx/tf/parse_gather.cpp deleted file mode 100644 index 965b8fa83..000000000 --- a/docker/rocm/migraphx/tf/parse_gather.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_gather : op_parser -{ - std::vector operators() const { return {{"GatherV2"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - int axis = args[2]->eval().at(); - return info.add_instruction(make_op("gather", {{"axis", axis}}), {args[0], args[1]}); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_generic_op.cpp b/docker/rocm/migraphx/tf/parse_generic_op.cpp deleted file mode 100644 index e459147fc..000000000 --- a/docker/rocm/migraphx/tf/parse_generic_op.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_generic_op : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const - { - return {{"All", "identity"}, - {"Identity", "identity"}, - {"LessEqual", "identity"}, - {"Relu", "relu"}, - {"Rsqrt", "rsqrt"}, - {"Tanh", "tanh"}, - {"StopGradient", "identity"}}; - } - - instruction_ref parse(const op_desc& opd, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - const std::vector& args) const - { - return info.add_instruction(make_op(opd.op_name), args); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_matmul.cpp b/docker/rocm/migraphx/tf/parse_matmul.cpp deleted file mode 100644 index 7ca4c52b4..000000000 --- a/docker/rocm/migraphx/tf/parse_matmul.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_matmul : op_parser -{ - std::vector operators() const - { - return {{"BatchMatMul"}, {"BatchMatMulV2"}, {"MatMul"}}; - } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - bool transa = false; - bool transb = false; - - if(contains(info.attributes, "transpose_a")) - { - transa = info.attributes.at("transpose_a").b(); - } - if(contains(info.attributes, "transpose_b")) - { - transb = info.attributes.at("transpose_b").b(); - } - - if(contains(info.attributes, "adj_x")) - { - transa = info.attributes.at("adj_x").b(); - } - if(contains(info.attributes, "adj_y")) - { - transb = info.attributes.at("adj_y").b(); - } - - std::vector perm(args[0]->get_shape().lens().size()); - std::iota(perm.begin(), perm.end(), int64_t{0}); - // swap the last two elements - std::iter_swap(perm.end() - 1, perm.end() - 2); - - auto l1 = (transa) - ? info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[0]) - : args[0]; - auto l2 = (transb) - ? info.add_instruction(make_op("transpose", {{"permutation", perm}}), args[1]) - : args[1]; - - return info.add_instruction(make_op("dot"), l1, l2); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_mean.cpp b/docker/rocm/migraphx/tf/parse_mean.cpp deleted file mode 100644 index d0c2b9952..000000000 --- a/docker/rocm/migraphx/tf/parse_mean.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_mean : op_parser -{ - std::vector operators() const { return {{"Mean"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - bool keep_dims = info.attributes.at("keep_dims").b(); - auto axes = args[1]->eval().get().to_vector(); - - auto ins = info.add_instruction(make_op("reduce_mean", {{"axes", axes}}), args[0]); - if(not keep_dims) - ins = info.add_instruction(make_op("squeeze", {{"axes", axes}}), ins); - return ins; - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_onehot.cpp b/docker/rocm/migraphx/tf/parse_onehot.cpp deleted file mode 100644 index 66f7d7d0a..000000000 --- a/docker/rocm/migraphx/tf/parse_onehot.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_onehot : op_parser -{ - std::vector operators() const { return {{"OneHot"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - size_t depth = args[1]->eval().at(); - - int64_t axis = -1; - float on_value = args[2]->eval().at(); - float off_value = args[3]->eval().at(); - - std::vector depth_input(depth * depth, off_value); - for(int i = 0; i < depth; i++) - { - depth_input[depth * i + i] = on_value; - } - - if(contains(info.attributes, "axis")) - axis = info.attributes.at("axis").i(); - if(axis == -1) - { - shape s{shape::float_type, {depth, depth}}; - auto l0 = info.add_literal({s, depth_input}); - return info.add_instruction(make_op("gather", {{"axis", 0}}), {l0, args[0]}); - } - MIGRAPHX_THROW("MIGraphX does not support axis != -1"); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_pack.cpp b/docker/rocm/migraphx/tf/parse_pack.cpp deleted file mode 100644 index 9766da1bc..000000000 --- a/docker/rocm/migraphx/tf/parse_pack.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_pack : op_parser -{ - std::vector operators() const { return {{"Pack"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& parser, - tf_parser::node_info info, - std::vector args) const - { - // reinterpret as unsqueeze with concat - std::vector unsqueezed_args; - int64_t axis = 0; - if(contains(info.attributes, "axis")) - axis = info.attributes.at("axis").i(); - size_t input_size = args.front()->get_shape().lens().size(); - if(axis > input_size) - { - MIGRAPHX_THROW("TF_PARSER: axis value of " + to_string(axis) + - " must be smaller than input size " + to_string(input_size)); - } - - std::transform( - args.begin(), - args.end(), - std::back_inserter(unsqueezed_args), - [&](instruction_ref arg) { - return info.add_instruction(make_op("unsqueeze", {{"axes", {axis}}}), arg); - }); - return parser.to_nhwc( - info.add_instruction(make_op("concat", {{"axis", axis}}), unsqueezed_args)); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_pad.cpp b/docker/rocm/migraphx/tf/parse_pad.cpp deleted file mode 100644 index b1d3a587d..000000000 --- a/docker/rocm/migraphx/tf/parse_pad.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_pad : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"Pad"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& parser, - const tf_parser::node_info& info, - std::vector args) const - { - size_t ndims = args.front()->get_shape().lens().size(); - - // in tf, the paddings are arranged as a 2d shape (ndims, 2), - // the last dim contains the left padding and right padding respectively - std::vector> pad_per_dim(ndims); - auto tf_padding = args[1]->eval().get().to_vector(); - for(size_t i = 0; i < 2 * ndims; i += 2) - { - pad_per_dim[i / 2].first = tf_padding[i]; - pad_per_dim[i / 2].second = tf_padding[i + 1]; - } - parser.reorder_data(pad_per_dim); - - std::vector pads(ndims * 2); - for(size_t i = 0; i < ndims; i++) - { - pads[i] = pad_per_dim[i].first; - pads[i + ndims] = pad_per_dim[i].second; - } - return info.add_instruction(make_op("pad", {{"pads", pads}}), args.front()); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_pooling.cpp b/docker/rocm/migraphx/tf/parse_pooling.cpp deleted file mode 100644 index 4baf09a86..000000000 --- a/docker/rocm/migraphx/tf/parse_pooling.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_pooling : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"AvgPool"}, {"MaxPool"}}; } - - instruction_ref parse(const op_desc& opd, - const tf_parser& parser, - tf_parser::node_info info, - std::vector args) const - { - if(not starts_with(opd.tf_name, "Max") and not starts_with(opd.tf_name, "Av")) - { - MIGRAPHX_THROW("tf pooling mode must be Max or Average"); - } - op::pooling op{starts_with(opd.tf_name, "Max") ? op::pooling_mode::max - : op::pooling_mode::average}; - - if(contains(info.attributes, "strides")) - { - std::vector stride; - copy(info.attributes.at("strides").list().i(), std::back_inserter(stride)); - parser.reorder_data(stride); - if(stride.size() != 4) - { - MIGRAPHX_THROW("strides should have 4 values"); - } - op.stride[0] = stride[2]; - op.stride[1] = stride[3]; - } - if(contains(info.attributes, "ksize")) - { - std::vector ksize; - copy(info.attributes.at("ksize").list().i(), std::back_inserter(ksize)); - parser.reorder_data(ksize); - if(ksize.size() != 4) - { - MIGRAPHX_THROW("ksize should have 4 values"); - } - op.lengths[0] = ksize[2]; - op.lengths[1] = ksize[3]; - } - - auto l0 = args[0]; - if(contains(info.attributes, "padding")) - { - const std::string& pad_mode = info.attributes.at("padding").s(); - if(pad_mode.find("SAME") != std::string::npos) - { - auto input_dims = l0->get_shape().lens(); - std::vector pads(input_dims.size()); - calculate_padding(0, pads, input_dims[2], op.stride[0], 1, op.lengths[0]); - calculate_padding(1, pads, input_dims[3], op.stride[1], 1, op.lengths[1]); - - op.padding = std::vector(pads.begin(), pads.end()); - } - } - return info.add_instruction(op, l0); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_relu6.cpp b/docker/rocm/migraphx/tf/parse_relu6.cpp deleted file mode 100644 index 75155b432..000000000 --- a/docker/rocm/migraphx/tf/parse_relu6.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_relu6 : op_parser -{ - bool transpose() const { return true; } - std::vector operators() const { return {{"Relu6"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - shape::type_t output_type = args[0]->get_shape().type(); - auto min_val = info.add_literal(migraphx::literal{migraphx::shape{output_type}, {0.0f}}); - auto max_val = info.add_literal(migraphx::literal{migraphx::shape{output_type}, {6.0f}}); - - return info.add_common_op("clip", args[0], min_val, max_val); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_reshape.cpp b/docker/rocm/migraphx/tf/parse_reshape.cpp deleted file mode 100644 index 4a1c7e697..000000000 --- a/docker/rocm/migraphx/tf/parse_reshape.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_reshape : op_parser -{ - std::vector operators() const { return {{"Reshape"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - if(args.size() != 2) - MIGRAPHX_THROW("reshape needs 2 arguments (input, new_shape)"); - auto s = args[1]->eval(); - std::vector dims; - s.visit([&](auto v) { copy(v, std::back_inserter(dims)); }); - return info.add_instruction(make_op("reshape", {{"dims", dims}}), args[0]); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_shape.cpp b/docker/rocm/migraphx/tf/parse_shape.cpp deleted file mode 100644 index bdc850ef3..000000000 --- a/docker/rocm/migraphx/tf/parse_shape.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_shape : op_parser -{ - std::vector operators() const { return {{"Shape"}}; } - - // Use a literal instruction to replace the shape since output of - // shape operator are literals in migraphx - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - std::vector arg_shape = args[0]->get_shape().lens(); - std::vector vec_shape(arg_shape.size()); - migraphx::shape s(migraphx::shape::int32_type, {arg_shape.size()}); - std::transform( - arg_shape.begin(), arg_shape.end(), vec_shape.begin(), [](auto i) { return i; }); - return info.add_literal(migraphx::literal{s, vec_shape}); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_slice.cpp b/docker/rocm/migraphx/tf/parse_slice.cpp deleted file mode 100644 index 4fd9be29c..000000000 --- a/docker/rocm/migraphx/tf/parse_slice.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_slice : op_parser -{ - std::vector operators() const { return {{"Slice"}}; } - - // Use a literal instruction to replace the shape since output of - // shape operator are literals in migraphx - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - auto starts = args[1]->eval().get().to_vector(); - auto size = args[2]->eval().get().to_vector(); - auto axes = args[0]->get_shape().lens(); - size_t num_axes = axes.size(); - - std::vector axes_int64(axes.begin(), axes.end()); - std::vector starts_int64(starts.begin(), starts.end()); - std::vector ends(num_axes); - std::vector op_axes(num_axes); - std::iota(op_axes.begin(), op_axes.end(), 0); - for(size_t i = 0; i < num_axes; i++) - { - if(size[i] == -1) - ends[i] = axes_int64[i]; - else - ends[i] = starts_int64[i] + size[i]; - } - auto op = make_op("slice", {{"starts", starts_int64}, {"ends", ends}, {"axes", op_axes}}); - return info.add_instruction(op, info.make_contiguous(args[0])); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_softmax.cpp b/docker/rocm/migraphx/tf/parse_softmax.cpp deleted file mode 100644 index a136e5c6f..000000000 --- a/docker/rocm/migraphx/tf/parse_softmax.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_softmax : op_parser -{ - std::vector operators() const { return {{"Softmax"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - int axis = -1; - auto num_dims = args[0]->get_shape().lens().size(); - if(contains(info.attributes, "axis")) - { - axis = static_cast(info.attributes.at("axis").i()); - } - - axis = tune_axis(num_dims, axis, "tf_parse_softmax"); - - return info.add_instruction(make_op("softmax", {{"axis", axis}}), - info.make_contiguous(args[0])); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_split.cpp b/docker/rocm/migraphx/tf/parse_split.cpp deleted file mode 100644 index 91aa23936..000000000 --- a/docker/rocm/migraphx/tf/parse_split.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_split : op_parser -{ - std::vector operators() const { return {{"Split"}, {"SplitV"}}; } - - std::vector parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - bool vector_as_input = args.size() == 3; - int num_outputs = 1; - auto axis_arg = args[0]; - auto input_arg = args[1]; - if(vector_as_input) - { - input_arg = args[0]; - axis_arg = args[2]; - } - - if(contains(info.attributes, "num_split")) - num_outputs = info.attributes.at("num_split").i(); - - std::vector splits(num_outputs); - std::vector slice_pos{0}; - if(vector_as_input) - { - splits = args[1]->eval().get().to_vector(); - num_outputs = splits.size(); - } - - assert(num_outputs > 0); - - if(num_outputs == 1) - return std::vector{ - info.add_instruction(make_op("identity"), input_arg)}; - - auto lens = input_arg->get_shape().lens(); - auto num_dims = lens.size(); - int axis = axis_arg->eval().at(); - - // ensure split is made evenly if "num_split" is used - assert(vector_as_input or lens[axis] % num_outputs == 0); - - auto split_size = lens[axis] / num_outputs; - - // push back first end point of slice - if(vector_as_input) - { - slice_pos.push_back(splits[0]); - } - else - { - slice_pos.push_back(split_size); - } - - // calculate remaining end points for each slice - for(auto i = 1; i < num_outputs; i++) - { - if(vector_as_input) - { - splits[i] += splits[i - 1]; - slice_pos.push_back(splits[i]); - } - else - { - slice_pos.push_back((i + 1) * split_size); - } - } - std::vector result; - for(auto i = 0; i < num_outputs; i++) - { - std::vector axes(num_dims); - std::iota(axes.begin(), axes.end(), 0); - std::vector starts(num_dims, 0); - std::vector ends(lens.begin(), lens.end()); - - starts[axis] = slice_pos[i]; - ends[axis] = slice_pos[i + 1]; - auto op = make_op("slice", {{"axes", axes}, {"starts", starts}, {"ends", ends}}); - result.push_back(info.add_instruction(op, input_arg)); - } - return result; - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_squeeze.cpp b/docker/rocm/migraphx/tf/parse_squeeze.cpp deleted file mode 100644 index 8936c0ac3..000000000 --- a/docker/rocm/migraphx/tf/parse_squeeze.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_squeeze : op_parser -{ - std::vector operators() const { return {{"Squeeze"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - auto input_dims = args[0]->get_shape().lens(); - auto axes = info.attributes.at("squeeze_dims").list().i(); - std::vector op_axes(axes.begin(), axes.end()); - - if(op_axes.empty()) // no squeeze_dims provided, remove any dim that equals 1 - { - for(size_t i = 0; i < input_dims.size(); i++) - { - if(input_dims.at(i) == 1) - { - op_axes.push_back(i); - } - } - } - return info.add_instruction(make_op("squeeze", {{"axes", op_axes}}), - info.make_contiguous(args[0])); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_stridedslice.cpp b/docker/rocm/migraphx/tf/parse_stridedslice.cpp deleted file mode 100644 index c161e8953..000000000 --- a/docker/rocm/migraphx/tf/parse_stridedslice.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_strideslice : op_parser -{ - std::vector operators() const { return {{"StridedSlice"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - tf_parser::node_info info, - std::vector args) const - { - auto starts = args[1]->eval().get().to_vector(); - auto ends = args[2]->eval().get().to_vector(); - auto l0 = args[0]; - size_t num_axes = l0->get_shape().lens().size(); - std::vector axes = l0->get_shape().lens(); - - std::vector op_starts(starts.begin(), starts.end()); - std::vector op_ends(ends.begin(), ends.end()); - std::vector op_axes(num_axes); - std::iota(op_axes.begin(), op_axes.end(), 0); - uint32_t begin_mask = 0; - uint32_t end_mask = 0; - uint32_t shrink_axis_mask = 0; - uint32_t bitwise_compare = 1; - std::vector squeeze_axes; - - if(contains(info.attributes, "begin_mask")) - begin_mask = static_cast(info.attributes.at("begin_mask").i()); - - if(contains(info.attributes, "end_mask")) - end_mask = static_cast(info.attributes.at("end_mask").i()); - - if(contains(info.attributes, "shrink_axis_mask")) - shrink_axis_mask = static_cast(info.attributes.at("shrink_axis_mask").i()); - - std::vector begin_axes = get_axes_from_mask(num_axes, begin_mask); - std::vector end_axes = get_axes_from_mask(num_axes, end_mask); - - for(size_t i = 0; i < num_axes; i++) - { - if(begin_axes.at(i) == 1) - { - op_starts.at(i) = 0; - } - if(end_axes.at(i) == 1) - { - op_ends.at(i) = axes.at(i); - } - } - - auto op = make_op("slice", {{"starts", op_starts}, {"ends", op_ends}, {"axes", op_axes}}); - auto l1 = info.add_instruction(op, l0); - if(shrink_axis_mask == 0) - return l1; - - for(size_t i = 0; i < num_axes; i++) - { - // the LSB corresponds to axis 0 when determining which axes to squeeze - if(((shrink_axis_mask >> i) & bitwise_compare) == 1) - squeeze_axes.push_back(i); - } - - return info.add_instruction(make_op("squeeze", {{"axes", squeeze_axes}}), l1); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/parse_transpose.cpp b/docker/rocm/migraphx/tf/parse_transpose.cpp deleted file mode 100644 index 9b306b97d..000000000 --- a/docker/rocm/migraphx/tf/parse_transpose.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -struct parse_transpose : op_parser -{ - std::vector operators() const { return {{"Transpose"}}; } - - instruction_ref parse(const op_desc& /*opd*/, - const tf_parser& /*parser*/, - const tf_parser::node_info& info, - std::vector args) const - { - auto perm = args[1]->eval().get().to_vector(); - std::vector dims(perm.begin(), perm.end()); - - return info.add_instruction(make_op("transpose", {{"permutation", dims}}), args.front()); - } -}; - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/resource_handle.proto b/docker/rocm/migraphx/tf/resource_handle.proto deleted file mode 100644 index a54d3d906..000000000 --- a/docker/rocm/migraphx/tf/resource_handle.proto +++ /dev/null @@ -1,30 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "ResourceHandle"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; - -// Protocol buffer representing a handle to a tensorflow resource. Handles are -// not valid across executions, but can be serialized back and forth from within -// a single run. -message ResourceHandleProto { - // Unique name for the device containing the resource. - string device = 1; - - // Container in which this resource is placed. - string container = 2; - - // Unique name of this resource. - string name = 3; - - // Hash code for the type of the resource. Is only valid in the same device - // and in the same execution. - uint64 hash_code = 4; - - // For debug-only, the name of the type pointed to by this handle, if - // available. - string maybe_type_name = 5; -}; diff --git a/docker/rocm/migraphx/tf/tensor.proto b/docker/rocm/migraphx/tf/tensor.proto deleted file mode 100644 index 5d4d66aed..000000000 --- a/docker/rocm/migraphx/tf/tensor.proto +++ /dev/null @@ -1,94 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "TensorProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "resource_handle.proto"; -import "tensor_shape.proto"; -import "types.proto"; - -// Protocol buffer representing a tensor. -message TensorProto { - DataType dtype = 1; - - // Shape of the tensor. TODO(touts): sort out the 0-rank issues. - TensorShapeProto tensor_shape = 2; - - // Only one of the representations below is set, one of "tensor_contents" and - // the "xxx_val" attributes. We are not using oneof because as oneofs cannot - // contain repeated fields it would require another extra set of messages. - - // Version number. - // - // In version 0, if the "repeated xxx" representations contain only one - // element, that element is repeated to fill the shape. This makes it easy - // to represent a constant Tensor with a single value. - int32 version_number = 3; - - // Serialized raw tensor content from either Tensor::AsProtoTensorContent or - // memcpy in tensorflow::grpc::EncodeTensorToByteBuffer. This representation - // can be used for all tensor types. The purpose of this representation is to - // reduce serialization overhead during RPC call by avoiding serialization of - // many repeated small items. - bytes tensor_content = 4; - - // Type specific representations that make it easy to create tensor protos in - // all languages. Only the representation corresponding to "dtype" can - // be set. The values hold the flattened representation of the tensor in - // row major order. - - // DT_HALF, DT_BFLOAT16. Note that since protobuf has no int16 type, we'll - // have some pointless zero padding for each value here. - repeated int32 half_val = 13 [packed = true]; - - // DT_FLOAT. - repeated float float_val = 5 [packed = true]; - - // DT_DOUBLE. - repeated double double_val = 6 [packed = true]; - - // DT_INT32, DT_INT16, DT_INT8, DT_UINT8. - repeated int32 int_val = 7 [packed = true]; - - // DT_STRING - repeated bytes string_val = 8; - - // DT_COMPLEX64. scomplex_val(2*i) and scomplex_val(2*i+1) are real - // and imaginary parts of i-th single precision complex. - repeated float scomplex_val = 9 [packed = true]; - - // DT_INT64 - repeated int64 int64_val = 10 [packed = true]; - - // DT_BOOL - repeated bool bool_val = 11 [packed = true]; - - // DT_COMPLEX128. dcomplex_val(2*i) and dcomplex_val(2*i+1) are real - // and imaginary parts of i-th double precision complex. - repeated double dcomplex_val = 12 [packed = true]; - - // DT_RESOURCE - repeated ResourceHandleProto resource_handle_val = 14; - - // DT_VARIANT - repeated VariantTensorDataProto variant_val = 15; - - // DT_UINT32 - repeated uint32 uint32_val = 16 [packed = true]; - - // DT_UINT64 - repeated uint64 uint64_val = 17 [packed = true]; -}; - -// Protocol buffer representing the serialization format of DT_VARIANT tensors. -message VariantTensorDataProto { - // Name of the type of objects being serialized. - string type_name = 1; - // Portions of the object that are not Tensors. - bytes metadata = 2; - // Tensors contained within objects being serialized. - repeated TensorProto tensors = 3; -} diff --git a/docker/rocm/migraphx/tf/tensor_shape.proto b/docker/rocm/migraphx/tf/tensor_shape.proto deleted file mode 100644 index 286156a01..000000000 --- a/docker/rocm/migraphx/tf/tensor_shape.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Protocol buffer representing the shape of tensors. - -syntax = "proto3"; -option cc_enable_arenas = true; -option java_outer_classname = "TensorShapeProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; - -package tensorflow; - -// Dimensions of a tensor. -message TensorShapeProto { - // One dimension of the tensor. - message Dim { - // Size of the tensor in that dimension. - // This value must be >= -1, but values of -1 are reserved for "unknown" - // shapes (values of -1 mean "unknown" dimension). Certain wrappers - // that work with TensorShapeProto may fail at runtime when deserializing - // a TensorShapeProto containing a dim value of -1. - int64 size = 1; - - // Optional name of the tensor dimension. - string name = 2; - }; - - // Dimensions of the tensor, such as {"input", 30}, {"output", 40} - // for a 30 x 40 2D tensor. If an entry has size -1, this - // corresponds to a dimension of unknown size. The names are - // optional. - // - // The order of entries in "dim" matters: It indicates the layout of the - // values in the tensor in-memory representation. - // - // The first entry in "dim" is the outermost dimension used to layout the - // values, the last entry is the innermost dimension. This matches the - // in-memory layout of RowMajor Eigen tensors. - // - // If "dim.size()" > 0, "unknown_rank" must be false. - repeated Dim dim = 2; - - // If true, the number of dimensions in the shape is unknown. - // - // If true, "dim.size()" must be 0. - bool unknown_rank = 3; -}; diff --git a/docker/rocm/migraphx/tf/tf.cpp b/docker/rocm/migraphx/tf/tf.cpp deleted file mode 100644 index 7b6c1322d..000000000 --- a/docker/rocm/migraphx/tf/tf.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -template -program parse_tf_from(const tf_options& options, Ts&&... xs) -{ - tf::tf_parser parser; - parser.is_nhwc = options.is_nhwc; - parser.batch_size = options.batch_size; - parser.map_input_dims = options.map_input_dims; - parser.output_node_names = options.output_node_names; - -#ifndef NDEBUG - // Log the program when it can't be parsed - try - { - parser.parse_from(std::forward(xs)...); - } - catch(...) - { - std::cerr << parser.prog << std::endl; - throw; - } -#else - parser.parse_from(std::forward(xs)...); -#endif - return std::move(parser.prog); -} - -program parse_tf(const std::string& name, const tf_options& options) -{ - std::fstream input(name.c_str(), std::ios::in | std::ios::binary); - return parse_tf_from(options, input); -} - -program parse_tf_buffer(const std::string& buffer, const tf_options& options) -{ - return parse_tf_from(options, buffer.data(), buffer.size()); -} - -program parse_tf_buffer(const void* data, std::size_t size, const tf_options& options) -{ - return parse_tf_from(options, data, size); -} - -std::vector get_tf_operators() { return tf::get_op_parsers(); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/tf_parser.cpp b/docker/rocm/migraphx/tf/tf_parser.cpp deleted file mode 100644 index a53c7b29e..000000000 --- a/docker/rocm/migraphx/tf/tf_parser.cpp +++ /dev/null @@ -1,605 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { -namespace tf { - -bool tf_parser::should_transpose(instruction_ref ins) const -{ - return is_nhwc and ins->get_shape().lens().size() == 4; -} - -instruction_ref tf_parser::to_nhwc(instruction_ref ins) const -{ - if(should_transpose(ins)) - return mm->add_instruction(make_op("transpose", {{"permutation", {0, 2, 3, 1}}}), ins); - return ins; -} - -instruction_ref tf_parser::to_nchw(instruction_ref ins) const -{ - if(should_transpose(ins)) - return mm->add_instruction(make_op("transpose", {{"permutation", {0, 3, 1, 2}}}), ins); - return ins; -} - -instruction_ref tf_parser::to_kcxy(instruction_ref ins) const -{ - return mm->add_instruction(make_op("transpose", {{"permutation", {3, 2, 0, 1}}}), ins); -} - -std::vector tf_parser::to_nchw(const std::vector& args) const -{ - std::vector result(args.size()); - std::transform( - args.begin(), args.end(), result.begin(), [&](auto ins) { return this->to_nchw(ins); }); - return result; -} - -std::vector tf_parser::to_nhwc(const std::vector& args) const -{ - std::vector result(args.size()); - std::transform( - args.begin(), args.end(), result.begin(), [&](auto ins) { return this->to_nhwc(ins); }); - return result; -} - -instruction_ref tf_parser::node_info::make_contiguous(instruction_ref ins) const -{ - if(ins->get_shape().standard()) - return ins; - else - return mm->add_instruction(make_op("contiguous"), ins); -} - -instruction_ref tf_parser::node_info::add_broadcastable_binary_op(const std::string& op_name, - instruction_ref arg0, - instruction_ref arg1) const -{ - return this->add_common_op(op_name, arg0, arg1); -} - -instruction_ref tf_parser::node_info::add_common_op(const std::string& op_name, - std::vector inputs) const -{ - return migraphx::add_common_op(*mm, make_op(op_name), std::move(inputs)); -} - -int64_t tf_parser::parse_axis(const int64_t dim, const size_t num_dims) const -{ - int64_t new_dim = dim; - if(is_nhwc and num_dims >= 4) - { - switch(dim) - { - case 0: new_dim = 0; break; - case 1: new_dim = 2; break; - case 2: new_dim = 3; break; - case 3: new_dim = 1; break; - default: break; - } - } - return new_dim; -} - -instruction_ref -tf_parser::node_info::add_instruction(const operation& op, - const std::vector& args) const -{ - return mm->add_instruction(op, args); -} - -instruction_ref tf_parser::node_info::add_literal(literal l) const -{ - return mm->add_literal(std::move(l)); -} - -std::vector get_axes_from_mask(const size_t num_axes, const uint32_t mask) -{ - uint32_t bitwise_compare = 1; - std::vector axes; - for(size_t i = 0; i < num_axes; i++) - { - // the LSB corresponds to axis 0 when determining which axes to begin - if(((mask >> i) & bitwise_compare) == 1) - axes.push_back(1); - else - axes.push_back(0); - } - return axes; -} - -tf_parser::tf_parser() -{ - // Add all registered op parsers - for(auto&& name : get_op_parsers()) - ops.emplace(name, get_op_parser(name)); -} - -static std::string get_name(const tensorflow::NodeDef& node) { return node.name(); } - -static tf_parser::node_map get_nodes(const tensorflow::GraphDef& graph, - std::vector& input_nodes) -{ - tf_parser::node_map result; - for(auto&& node : graph.node()) - { - auto node_name = get_name(node); - // assume each node in graph has an associated name - if(node_name.empty()) - MIGRAPHX_THROW("tf node with no name found"); - result[node_name] = node; - if(node.op() == "Placeholder") - { - input_nodes.push_back(node); - } - } - return result; -} - -static tf_parser::attribute_map get_attributes(const tensorflow::NodeDef& node) -{ - tf_parser::attribute_map result; - for(auto&& attr : node.attr()) - { - result[attr.first] = attr.second; - } - - return result; -} - -static std::vector parse_dims(const tensorflow::TensorShapeProto& s) -{ - std::vector dims; - auto input_dims = s.dim(); - std::transform(input_dims.begin(), - input_dims.end(), - std::back_inserter(dims), - [](const tensorflow::TensorShapeProto_Dim& dim) { return dim.size(); }); - return dims; -} - -template -static std::vector get_data_vals(const google::protobuf::RepeatedField& data, - const size_t& shape_size) -{ - std::vector data_vals(shape_size); - // check if shape has enough data values given existing fields - if(data.size() == 1) - { - std::fill(data_vals.begin(), data_vals.end(), data[0]); - } - else - copy(data.begin(), data.end(), data_vals.begin()); - return data_vals; -} - -template -static literal -create_literal(shape::type_t shape_type, const std::vector& dims, std::vector data) -{ - // assume if explicit value is mentioned in protobuf and dim size <= 1, treat as scalar - if(dims.empty() or (dims.size() == 1 and dims.front() == 1)) - return literal{{shape_type}, data}; - return literal{{shape_type, dims}, data}; -} - -static bool is_valid_op(const tensorflow::NodeDef& node) -{ - std::vector ignored{"NoOp", "Assert"}; - return none_of(ignored, [&](const auto& op) { - const auto& name = get_name(node); - return node.op() == op or contains(name, op); - }); -} - -std::vector tf_parser::find_outputs() const -{ - std::unordered_set inputs; - for(auto&& p : nodes) - { - auto&& node = p.second; - std::copy(node.input().begin(), node.input().end(), std::inserter(inputs, inputs.end())); - } - std::vector outputs; - for(auto&& p : nodes) - { - const auto& name = p.first; - const auto& node = p.second; - if(not is_valid_op(node)) - continue; - // control flow related, ignore this node - if(contains(name, "^")) - continue; - // literals are valid ops, but they are not outputs unless specified - if(node.op() == "Const") - continue; - if(inputs.count(name) == 0) - outputs.push_back(name); - } - return outputs; -} - -void tf_parser::parse_graph(const tensorflow::GraphDef& graph) -{ - nodes = get_nodes(graph, input_nodes); - for(auto&& input : input_nodes) - { - const std::string& name = input.name(); - attribute_map input_attrs = get_attributes(input); - shape::type_t shape_type = parse_type(input_attrs.at("dtype").type()); - std::vector dims = parse_dims(input_attrs.at("shape").shape()); - - if(contains(map_input_dims, name)) - { - dims = map_input_dims.at(name); - } - else - { - if(is_nhwc and dims.size() >= 4) - { - this->reorder_data(dims); - } - std::transform(dims.begin(), dims.end(), dims.begin(), [&](auto dim) { - return static_cast(dim) <= 0 ? batch_size : dim; - }); - } - - shape s = shape{shape_type, dims}; - instructions[name] = to_nhwc(mm->add_parameter(name, s)); - } - for(auto&& p : nodes) - { - this->parse_node(p.first); - } - if(mm->size() == 0) - return; - - // Needs to add a ret instruction at the end of - // the program - if(output_node_names.empty()) - { - output_node_names = find_outputs(); - } - - std::vector output_ins; - std::transform(output_node_names.begin(), - output_node_names.end(), - std::back_inserter(output_ins), - [&](auto output_name) { - if(not contains(instructions, output_name)) - MIGRAPHX_THROW("PARSE_TF: output name " + output_name + - " not found in graph!"); - return this->to_nchw(instructions[output_name]); - }); - mm->add_return(output_ins); -} - -void tf_parser::parse_node(const std::string& name) -{ - if(instructions.count(name) == 0) - { - auto&& node = nodes.at(name); - if(not is_valid_op(node)) - return; - std::vector args; - for(auto&& input : node.input()) - { - // control dependencies (signified by ^ before the name) are ignored - if(contains(input, "^")) - continue; - std::string input_name = input; - // if input has trailing `:0` index then remove it - auto multi_out_idx = input.find(':'); - if(multi_out_idx != std::string::npos and input.substr(multi_out_idx + 1) == "0") - { - input_name = input.substr(0, multi_out_idx); - } - if(nodes.count(input_name) > 0) - { - // input was from a node with multiple outputs - if(contains(input_name, ':')) - { - input_name.resize(input.find(':')); - } - else - { - input_name = get_name(nodes.at(input_name)); - } - assert(name != input_name); - this->parse_node(input_name); - args.push_back(instructions.at(input_name)); - } - else - { - args.push_back(instructions.at(input_name)); - } - } - std::vector result; - if(ops.count(node.op()) == 0) - { - result.push_back(mm->add_instruction(op::unknown{node.op()}, args)); - } - else - { - result = ops[node.op()](*this, {get_attributes(node), node.op(), mm}, args); - } - assert(not result.empty()); - // First output has no ":" delimiter - instructions[name] = result.front(); - for(size_t i = 1; i < result.size(); i++) - { - instructions[name + ":" + std::to_string(i)] = result.at(i); - } - } -} - -void tf_parser::parse_from(std::istream& is) -{ - tensorflow::GraphDef graph; - if(graph.ParseFromIstream(&is)) - { - this->parse_graph(graph); - } - else - { - throw std::runtime_error("Failed reading tf file"); - } -} - -void tf_parser::parse_from(const void* data, std::size_t size) -{ - tensorflow::GraphDef graph; - if(graph.ParseFromArray(data, size)) - { - this->parse_graph(graph); - } - else - { - throw std::runtime_error("Failed reading tf buffer array"); - } -} - -shape::type_t tf_parser::parse_type(const tensorflow::DataType t) const -{ - shape::type_t shape_type{}; - switch(t) - { - case tensorflow::DataType::DT_FLOAT: shape_type = shape::float_type; break; - case tensorflow::DataType::DT_DOUBLE: shape_type = shape::double_type; break; - case tensorflow::DataType::DT_INT32: shape_type = shape::int32_type; break; - case tensorflow::DataType::DT_INT16: shape_type = shape::int16_type; break; - case tensorflow::DataType::DT_INT8: shape_type = shape::int8_type; break; - case tensorflow::DataType::DT_INT64: shape_type = shape::int64_type; break; - case tensorflow::DataType::DT_UINT16: shape_type = shape::uint16_type; break; - case tensorflow::DataType::DT_HALF: shape_type = shape::half_type; break; - case tensorflow::DataType::DT_UINT32: shape_type = shape::uint32_type; break; - case tensorflow::DataType::DT_UINT64: shape_type = shape::uint64_type; break; - - case tensorflow::DataType::DT_INVALID: - case tensorflow::DataType::DT_UINT8: - case tensorflow::DataType::DT_STRING: - case tensorflow::DataType::DT_COMPLEX64: - case tensorflow::DataType::DT_BOOL: - case tensorflow::DataType::DT_QINT8: - case tensorflow::DataType::DT_QUINT8: - case tensorflow::DataType::DT_QINT32: - case tensorflow::DataType::DT_BFLOAT16: - case tensorflow::DataType::DT_QINT16: - case tensorflow::DataType::DT_QUINT16: - case tensorflow::DataType::DT_COMPLEX128: - case tensorflow::DataType::DT_RESOURCE: - case tensorflow::DataType::DT_VARIANT: - // tf pb should not use these types - case tensorflow::DataType::DT_FLOAT_REF: - case tensorflow::DataType::DT_DOUBLE_REF: - case tensorflow::DataType::DT_INT32_REF: - case tensorflow::DataType::DT_UINT8_REF: - case tensorflow::DataType::DT_INT16_REF: - case tensorflow::DataType::DT_INT8_REF: - case tensorflow::DataType::DT_STRING_REF: - case tensorflow::DataType::DT_COMPLEX64_REF: - case tensorflow::DataType::DT_INT64_REF: - case tensorflow::DataType::DT_BOOL_REF: - case tensorflow::DataType::DT_QINT8_REF: - case tensorflow::DataType::DT_QUINT8_REF: - case tensorflow::DataType::DT_QINT32_REF: - case tensorflow::DataType::DT_BFLOAT16_REF: - case tensorflow::DataType::DT_QINT16_REF: - case tensorflow::DataType::DT_QUINT16_REF: - case tensorflow::DataType::DT_UINT16_REF: - case tensorflow::DataType::DT_COMPLEX128_REF: - case tensorflow::DataType::DT_HALF_REF: - case tensorflow::DataType::DT_RESOURCE_REF: - case tensorflow::DataType::DT_VARIANT_REF: - case tensorflow::DataType::DT_UINT32_REF: - case tensorflow::DataType::DT_UINT64_REF: - case tensorflow::DataType::DataType_INT_MAX_SENTINEL_DO_NOT_USE_: - case tensorflow::DataType::DataType_INT_MIN_SENTINEL_DO_NOT_USE_: break; - } - return shape_type; -} - -literal tf_parser::parse_tensor(const tensorflow::TensorProto& t) const -{ - std::vector dims = parse_dims(t.tensor_shape()); - size_t shape_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); - if(not t.tensor_content().empty()) // has raw data - { - const std::string& s = t.tensor_content(); - switch(t.dtype()) - { - case tensorflow::DataType::DT_FLOAT: return literal{{shape::float_type, dims}, s.data()}; - case tensorflow::DataType::DT_BOOL: - case tensorflow::DataType::DT_INT8: return literal{{shape::int8_type, dims}, s.data()}; - case tensorflow::DataType::DT_UINT16: - case tensorflow::DataType::DT_INT16: return literal{{shape::int16_type, dims}, s.data()}; - case tensorflow::DataType::DT_INT32: return literal{{shape::int32_type, dims}, s.data()}; - case tensorflow::DataType::DT_INT64: return literal{{shape::int64_type, dims}, s.data()}; - case tensorflow::DataType::DT_HALF: return literal{{shape::half_type, dims}, s.data()}; - case tensorflow::DataType::DT_DOUBLE: return literal{{shape::double_type, dims}, s.data()}; - case tensorflow::DataType::DT_INVALID: - case tensorflow::DataType::DT_UINT8: - case tensorflow::DataType::DT_STRING: - case tensorflow::DataType::DT_UINT32: - case tensorflow::DataType::DT_UINT64: - case tensorflow::DataType::DT_COMPLEX64: - case tensorflow::DataType::DT_COMPLEX128: - case tensorflow::DataType::DT_QINT8: - case tensorflow::DataType::DT_QUINT8: - case tensorflow::DataType::DT_QINT32: - case tensorflow::DataType::DT_BFLOAT16: - case tensorflow::DataType::DT_QINT16: - case tensorflow::DataType::DT_QUINT16: - case tensorflow::DataType::DT_RESOURCE: - case tensorflow::DataType::DT_VARIANT: - case tensorflow::DataType::DT_FLOAT_REF: - case tensorflow::DataType::DT_DOUBLE_REF: - case tensorflow::DataType::DT_INT32_REF: - case tensorflow::DataType::DT_UINT8_REF: - case tensorflow::DataType::DT_INT16_REF: - case tensorflow::DataType::DT_INT8_REF: - case tensorflow::DataType::DT_STRING_REF: - case tensorflow::DataType::DT_COMPLEX64_REF: - case tensorflow::DataType::DT_INT64_REF: - case tensorflow::DataType::DT_BOOL_REF: - case tensorflow::DataType::DT_QINT8_REF: - case tensorflow::DataType::DT_QUINT8_REF: - case tensorflow::DataType::DT_QINT32_REF: - case tensorflow::DataType::DT_BFLOAT16_REF: - case tensorflow::DataType::DT_QINT16_REF: - case tensorflow::DataType::DT_QUINT16_REF: - case tensorflow::DataType::DT_UINT16_REF: - case tensorflow::DataType::DT_COMPLEX128_REF: - case tensorflow::DataType::DT_HALF_REF: - case tensorflow::DataType::DT_RESOURCE_REF: - case tensorflow::DataType::DT_VARIANT_REF: - case tensorflow::DataType::DT_UINT32_REF: - case tensorflow::DataType::DT_UINT64_REF: - case tensorflow::DataType::DataType_INT_MAX_SENTINEL_DO_NOT_USE_: - case tensorflow::DataType::DataType_INT_MIN_SENTINEL_DO_NOT_USE_: - throw std::runtime_error(""); - } - MIGRAPHX_THROW("Invalid tensor type"); - } - switch(t.dtype()) - { - case tensorflow::DataType::DT_FLOAT: - return create_literal(shape::float_type, dims, get_data_vals(t.float_val(), shape_size)); - case tensorflow::DataType::DT_INT8: - return create_literal(shape::int8_type, dims, get_data_vals(t.int_val(), shape_size)); - case tensorflow::DataType::DT_UINT16: - return create_literal(shape::uint16_type, dims, get_data_vals(t.int_val(), shape_size)); - case tensorflow::DataType::DT_INT16: - return create_literal(shape::int16_type, dims, get_data_vals(t.int_val(), shape_size)); - case tensorflow::DataType::DT_INT32: - return create_literal(shape::int32_type, dims, get_data_vals(t.int_val(), shape_size)); - case tensorflow::DataType::DT_INT64: - return create_literal(shape::int64_type, dims, get_data_vals(t.int64_val(), shape_size)); - case tensorflow::DataType::DT_BOOL: - return create_literal(shape::int32_type, dims, get_data_vals(t.bool_val(), shape_size)); - case tensorflow::DataType::DT_HALF: { - std::vector data_int32 = get_data_vals(t.half_val(), shape_size); - std::vector data_uint16(data_int32.begin(), data_int32.end()); - std::vector data_half; - std::transform(data_uint16.begin(), - data_uint16.end(), - std::back_inserter(data_half), - [](uint16_t raw_val) { return *reinterpret_cast(&raw_val); }); - return create_literal(shape::half_type, dims, data_half); - } - case tensorflow::DataType::DT_DOUBLE: - return literal{{shape::double_type, dims}, get_data_vals(t.double_val(), shape_size)}; - case tensorflow::DataType::DT_INVALID: - case tensorflow::DataType::DT_UINT8: - case tensorflow::DataType::DT_STRING: - case tensorflow::DataType::DT_UINT32: - case tensorflow::DataType::DT_UINT64: - case tensorflow::DataType::DT_COMPLEX64: - case tensorflow::DataType::DT_COMPLEX128: - case tensorflow::DataType::DT_QINT8: - case tensorflow::DataType::DT_QUINT8: - case tensorflow::DataType::DT_QINT32: - case tensorflow::DataType::DT_BFLOAT16: - case tensorflow::DataType::DT_QINT16: - case tensorflow::DataType::DT_QUINT16: - case tensorflow::DataType::DT_RESOURCE: - case tensorflow::DataType::DT_VARIANT: - case tensorflow::DataType::DT_FLOAT_REF: - case tensorflow::DataType::DT_DOUBLE_REF: - case tensorflow::DataType::DT_INT32_REF: - case tensorflow::DataType::DT_UINT8_REF: - case tensorflow::DataType::DT_INT16_REF: - case tensorflow::DataType::DT_INT8_REF: - case tensorflow::DataType::DT_STRING_REF: - case tensorflow::DataType::DT_COMPLEX64_REF: - case tensorflow::DataType::DT_INT64_REF: - case tensorflow::DataType::DT_BOOL_REF: - case tensorflow::DataType::DT_QINT8_REF: - case tensorflow::DataType::DT_QUINT8_REF: - case tensorflow::DataType::DT_QINT32_REF: - case tensorflow::DataType::DT_BFLOAT16_REF: - case tensorflow::DataType::DT_QINT16_REF: - case tensorflow::DataType::DT_QUINT16_REF: - case tensorflow::DataType::DT_UINT16_REF: - case tensorflow::DataType::DT_COMPLEX128_REF: - case tensorflow::DataType::DT_HALF_REF: - case tensorflow::DataType::DT_RESOURCE_REF: - case tensorflow::DataType::DT_VARIANT_REF: - case tensorflow::DataType::DT_UINT32_REF: - case tensorflow::DataType::DT_UINT64_REF: - case tensorflow::DataType::DataType_INT_MAX_SENTINEL_DO_NOT_USE_: - case tensorflow::DataType::DataType_INT_MIN_SENTINEL_DO_NOT_USE_: throw std::runtime_error(""); - } - MIGRAPHX_THROW("Invalid tensor type"); -} - -} // namespace tf -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/tf/types.proto b/docker/rocm/migraphx/tf/types.proto deleted file mode 100644 index 03835d1b9..000000000 --- a/docker/rocm/migraphx/tf/types.proto +++ /dev/null @@ -1,75 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "TypesProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; - -// LINT.IfChange -enum DataType { - // Not a legal value for DataType. Used to indicate a DataType field - // has not been set. - DT_INVALID = 0; - - // Data types that all computation devices are expected to be - // capable to support. - DT_FLOAT = 1; - DT_DOUBLE = 2; - DT_INT32 = 3; - DT_UINT8 = 4; - DT_INT16 = 5; - DT_INT8 = 6; - DT_STRING = 7; - DT_COMPLEX64 = 8; // Single-precision complex - DT_INT64 = 9; - DT_BOOL = 10; - DT_QINT8 = 11; // Quantized int8 - DT_QUINT8 = 12; // Quantized uint8 - DT_QINT32 = 13; // Quantized int32 - DT_BFLOAT16 = 14; // Float32 truncated to 16 bits. Only for cast ops. - DT_QINT16 = 15; // Quantized int16 - DT_QUINT16 = 16; // Quantized uint16 - DT_UINT16 = 17; - DT_COMPLEX128 = 18; // Double-precision complex - DT_HALF = 19; - DT_RESOURCE = 20; - DT_VARIANT = 21; // Arbitrary C++ data types - DT_UINT32 = 22; - DT_UINT64 = 23; - - // Do not use! These are only for parameters. Every enum above - // should have a corresponding value below (verified by types_test). - DT_FLOAT_REF = 101; - DT_DOUBLE_REF = 102; - DT_INT32_REF = 103; - DT_UINT8_REF = 104; - DT_INT16_REF = 105; - DT_INT8_REF = 106; - DT_STRING_REF = 107; - DT_COMPLEX64_REF = 108; - DT_INT64_REF = 109; - DT_BOOL_REF = 110; - DT_QINT8_REF = 111; - DT_QUINT8_REF = 112; - DT_QINT32_REF = 113; - DT_BFLOAT16_REF = 114; - DT_QINT16_REF = 115; - DT_QUINT16_REF = 116; - DT_UINT16_REF = 117; - DT_COMPLEX128_REF = 118; - DT_HALF_REF = 119; - DT_RESOURCE_REF = 120; - DT_VARIANT_REF = 121; - DT_UINT32_REF = 122; - DT_UINT64_REF = 123; -} -// LINT.ThenChange( -// https://www.tensorflow.org/code/tensorflow/c/c_api.h, -// https://www.tensorflow.org/code/tensorflow/go/tensor.go, -// https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc, -// https://www.tensorflow.org/code/tensorflow/core/framework/types.h, -// https://www.tensorflow.org/code/tensorflow/core/framework/types.cc, -// https://www.tensorflow.org/code/tensorflow/python/framework/dtypes.py, -// https://www.tensorflow.org/code/tensorflow/python/framework/function.py) diff --git a/docker/rocm/migraphx/tf/versions.proto b/docker/rocm/migraphx/tf/versions.proto deleted file mode 100644 index dd2ec5523..000000000 --- a/docker/rocm/migraphx/tf/versions.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "VersionsProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; - -// Version information for a piece of serialized data -// -// There are different types of versions for each type of data -// (GraphDef, etc.), but they all have the same common shape -// described here. -// -// Each consumer has "consumer" and "min_producer" versions (specified -// elsewhere). A consumer is allowed to consume this data if -// -// producer >= min_producer -// consumer >= min_consumer -// consumer not in bad_consumers -// -message VersionDef { - // The version of the code that produced this data. - int32 producer = 1; - - // Any consumer below this version is not allowed to consume this data. - int32 min_consumer = 2; - - // Specific consumer versions which are disallowed (e.g. due to bugs). - repeated int32 bad_consumers = 3; -}; diff --git a/docker/rocm/migraphx/tmp_dir.cpp b/docker/rocm/migraphx/tmp_dir.cpp deleted file mode 100644 index f4188a1d0..000000000 --- a/docker/rocm/migraphx/tmp_dir.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -// cppcheck-suppress definePrefix -#define WIN32_LEAN_AND_MEAN -#include -#undef getpid -// cppcheck-suppress [definePrefix, defineUpperCase] -#define getpid _getpid -#else -#include -#include -#endif - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DEBUG_SAVE_TEMP_DIR) - -std::string random_string(std::string::size_type length) -{ - static const std::string& chars = "0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - std::mt19937 rg{std::random_device{}()}; - std::uniform_int_distribution pick(0, chars.length() - 1); - - std::string str(length, 0); - std::generate(str.begin(), str.end(), [&] { return chars[pick(rg)]; }); - - return str; -} - -std::string unique_string(const std::string& prefix) -{ - auto pid = getpid(); - auto tid = std::this_thread::get_id(); - auto clk = std::chrono::steady_clock::now().time_since_epoch().count(); - std::stringstream ss; - ss << std::hex << prefix << "-" << pid << "-" << tid << "-" << clk << "-" << random_string(16); - return ss.str(); -} - -tmp_dir::tmp_dir(std::string_view prefix) - : path(fs::temp_directory_path() / - unique_string(prefix.empty() ? "migraphx" : "migraphx-" + std::string{prefix})) -{ - fs::create_directories(this->path); -} - -void tmp_dir::execute(std::string_view cmd, const std::vector& args) const -{ - process{cmd, args}.cwd(this->path).exec(); -} - -tmp_dir::~tmp_dir() -{ - if(not enabled(MIGRAPHX_DEBUG_SAVE_TEMP_DIR{})) - { - constexpr int max_retries_count = 5; - for([[maybe_unused]] auto count : range(max_retries_count)) - { - std::error_code ec; - fs::remove_all(path, ec); - if(not ec) - break; - std::cerr << "Failed to remove " << path << ": " << ec.message() << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(125)); - } - } -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/truncate_float.cpp b/docker/rocm/migraphx/truncate_float.cpp deleted file mode 100644 index 15f807684..000000000 --- a/docker/rocm/migraphx/truncate_float.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -static void -quantize_module(module& m, const std::vector& ins_names, shape::type_t float_type) -{ - for(auto ins : iterator_for(m)) - { - // instructions are not in the set to be quantized - if(not(contains(ins_names, ins->name()) or contains(ins_names, "all"))) - continue; - - // skip return and convert instructions - if(contains({"@return", "convert"}, ins->name())) - continue; - - if(ins->inputs().empty()) - continue; - - auto mod_inputs = ins->module_inputs(); - auto s = ins->get_shape(); - // Convert each of the inputs that are floating point to float type - auto inputs = ins->inputs(); - std::transform(inputs.begin(), inputs.end(), inputs.begin(), [&](auto input) { - auto input_type = input->get_shape().type(); - if(input_type != shape::float_type and input_type != shape::double_type) - return input; - return m.insert_instruction( - ins, make_op("convert", {{"target_type", float_type}}), input); - }); - - // Insert quantized ins - auto converted_ins = m.insert_instruction(ins, ins->get_operator(), inputs, mod_inputs); - - // tuple can't be directly converted, get_tuple_elem needs conversion - if(ins->get_shape().type() == shape::tuple_type) - { - auto outputs = ins->outputs(); - std::transform( - outputs.begin(), outputs.end(), outputs.begin(), [&](const auto gte_ins) { - auto gte_ins_float_type = - m.insert_instruction(ins, gte_ins->get_operator(), converted_ins); - // Convert back to output type after quantizing - auto gte_converted = m.insert_instruction( - ins, - make_op("convert", {{"target_type", gte_ins->get_shape().type()}}), - gte_ins_float_type); - // Replace output instruction - return m.replace_instruction(gte_ins, gte_converted); - }); - } - else - { - // Convert back to original type after quantizing - if(mod_inputs.empty()) - { - converted_ins = m.insert_instruction( - ins, make_op("convert", {{"target_type", s.type()}}), converted_ins); - } - // Replace original instruction - m.replace_instruction(ins, converted_ins); - } - } -} - -void truncate_float_pass::apply(module& m) const { quantize_module(m, ins_names, float_type); } - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/value.cpp b/docker/rocm/migraphx/value.cpp deleted file mode 100644 index dc6c4666e..000000000 --- a/docker/rocm/migraphx/value.cpp +++ /dev/null @@ -1,573 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -struct value_base_impl : cloneable -{ - virtual value::type_t get_type() { return value::null_type; } -#define MIGRAPHX_VALUE_GENERATE_BASE_FUNCTIONS(vt, cpp_type) \ - virtual const cpp_type* if_##vt() const { return nullptr; } - MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_BASE_FUNCTIONS) - virtual std::vector* if_array() { return nullptr; } - virtual std::unordered_map* if_object() { return nullptr; } - virtual value_base_impl* if_value() const { return nullptr; } - value_base_impl() = default; - value_base_impl(const value_base_impl&) = default; - value_base_impl& operator=(const value_base_impl&) = default; - virtual ~value_base_impl() override {} -}; - -#define MIGRAPHX_VALUE_GENERATE_BASE_TYPE(vt, cpp_type) \ - struct vt##_value_holder : value_base_impl::share \ - { \ - vt##_value_holder(cpp_type d) : data(std::move(d)) {} \ - virtual value::type_t get_type() override { return value::vt##_type; } \ - virtual const cpp_type* if_##vt() const override { return &data; } \ - cpp_type data; \ - }; -MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_BASE_TYPE) - -struct array_value_holder : value_base_impl::derive -{ - array_value_holder() {} - array_value_holder(std::vector d) : data(std::move(d)) {} - virtual value::type_t get_type() override { return value::array_type; } - virtual std::vector* if_array() override { return &data; } - std::vector data; -}; - -struct object_value_holder : value_base_impl::derive -{ - object_value_holder() {} - object_value_holder(std::vector d, std::unordered_map l) - : data(std::move(d)), lookup(std::move(l)) - { - } - virtual value::type_t get_type() override { return value::object_type; } - virtual std::vector* if_array() override { return &data; } - virtual std::unordered_map* if_object() override { return &lookup; } - std::vector data; - std::unordered_map lookup; -}; - -value::value(const value& rhs) : x(rhs.x ? rhs.x->clone() : nullptr), key(rhs.key) {} -value& value::operator=(value rhs) -{ - std::swap(rhs.x, x); - if(not rhs.key.empty()) - std::swap(rhs.key, key); - return *this; -} - -void set_vector(std::shared_ptr& x, - const std::vector& v, - bool array_on_empty = true) -{ - if(v.empty()) - { - if(array_on_empty) - x = std::make_shared(); - else - x = std::make_shared(); - return; - } - if(v.front().get_key().empty()) - { - x = std::make_shared(v); - } - else - { - std::unordered_map lookup; - std::size_t i = 0; - for(auto&& e : v) - { - lookup[e.get_key()] = i; - i++; - } - x = std::make_shared(v, lookup); - } -} - -value::value(const std::initializer_list& i) : x(nullptr) -{ - if(i.size() == 2 and i.begin()->is_string() and i.begin()->get_key().empty()) - { - key = i.begin()->get_string(); - auto r = (i.begin() + 1)->x; - x = r ? r->clone() : nullptr; - return; - } - set_vector(x, std::vector(i.begin(), i.end())); -} - -value::value(const std::vector& v, bool array_on_empty) : x(nullptr) -{ - set_vector(x, v, array_on_empty); -} - -value::value(const std::unordered_map& m) - : value(std::vector(m.begin(), m.end()), false) -{ -} - -value::value(const std::string& pkey, const std::vector& v, bool array_on_empty) - : x(nullptr), key(pkey) -{ - set_vector(x, v, array_on_empty); -} - -value::value(const std::string& pkey, const std::unordered_map& m) - : value(pkey, std::vector(m.begin(), m.end()), false) -{ -} - -value::value(const std::string& pkey, std::nullptr_t) : x(nullptr), key(pkey) {} - -value::value(std::nullptr_t) : x(nullptr) {} - -value::value(const std::string& pkey, const value& rhs) - : x(rhs.x ? rhs.x->clone() : nullptr), key(pkey) -{ -} - -value::value(const std::string& pkey, const char* i) : value(pkey, std::string(i)) {} -value::value(const char* i) : value(std::string(i)) {} - -#define MIGRAPHX_VALUE_GENERATE_DEFINE_METHODS(vt, cpp_type) \ - value::value(cpp_type i) : x(std::make_shared(std::move(i))) {} \ - value::value(const std::string& pkey, cpp_type i) \ - : x(std::make_shared(std::move(i))), key(pkey) \ - { \ - } \ - value& value::operator=(cpp_type rhs) \ - { \ - x = std::make_shared(std::move(rhs)); \ - return *this; \ - } \ - bool value::is_##vt() const { return x ? x->get_type() == vt##_type : false; } \ - const cpp_type& value::get_##vt() const \ - { \ - auto* r = this->if_##vt(); \ - assert(r); \ - return *r; \ - } \ - const cpp_type* value::if_##vt() const { return x ? x->if_##vt() : nullptr; } -MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_DEFINE_METHODS) - -value& value::operator=(const char* c) -{ - *this = std::string{c}; - return *this; -} - -value& value::operator=(std::nullptr_t) -{ - x = nullptr; - return *this; -} - -value& value::operator=(const std::initializer_list& i) -{ - value rhs = i; - std::swap(rhs.x, x); - return *this; -} - -bool value::is_array() const { return x ? x->get_type() == array_type : false; } -const std::vector& value::value::get_array() const -{ - const auto* r = this->if_array(); - assert(r); - return *r; -} -const std::vector* value::if_array() const { return x ? x->if_array() : nullptr; } - -bool value::is_object() const { return x ? x->get_type() == object_type : false; } -const std::vector& value::get_object() const -{ - const auto* r = this->if_object(); - assert(r); - return *r; -} -const std::vector* value::if_object() const -{ - auto* r = x ? x->if_array() : nullptr; - assert(r == nullptr or - std::none_of(r->begin(), r->end(), [](auto&& v) { return v.get_key().empty(); })); - return r; -} - -bool value::is_null() const { return x == nullptr; } - -const std::string& value::get_key() const { return key; } - -std::vector* if_array_impl(const std::shared_ptr& x) -{ - if(x == nullptr) - return nullptr; - return x->if_array(); -} - -std::vector& get_array_impl(const std::shared_ptr& x) -{ - auto* a = if_array_impl(x); - assert(a); - return *a; -} - -std::vector& get_array_throw(const std::shared_ptr& x) -{ - auto* a = if_array_impl(x); - if(a == nullptr) - MIGRAPHX_THROW("Expected an array or object"); - return *a; -} - -template -T* find_impl(const std::shared_ptr& x, const std::string& key, T* end) -{ - auto* a = if_array_impl(x); - if(a == nullptr) - return end; - auto* lookup = x->if_object(); - if(lookup == nullptr) - return end; - auto it = lookup->find(key); - if(it == lookup->end()) - return end; - return std::addressof((*a)[it->second]); -} - -value* value::find(const std::string& pkey) { return find_impl(x, pkey, this->end()); } - -const value* value::find(const std::string& pkey) const { return find_impl(x, pkey, this->end()); } -bool value::contains(const std::string& pkey) const -{ - const auto* it = find(pkey); - if(it == nullptr) - return false; - if(it == end()) - return false; - return true; -} -std::size_t value::size() const -{ - const auto* a = if_array_impl(x); - if(a == nullptr) - return 0; - return a->size(); -} -bool value::empty() const { return size() == 0; } -const value* value::data() const -{ - auto* a = if_array_impl(x); - if(a == nullptr) - return nullptr; - return a->data(); -} -value* value::data() -{ - auto* a = if_array_impl(x); - if(a == nullptr) - return nullptr; - return a->data(); -} -value* value::begin() -{ - // cppcheck-suppress assertWithSideEffect - assert(data() or empty()); - return data(); -} -const value* value::begin() const -{ - assert(data() or empty()); - return data(); -} -value* value::end() { return begin() + size(); } -const value* value::end() const { return begin() + size(); } - -value& value::front() -{ - assert(this->size() > 0); - return *begin(); -} -const value& value::front() const -{ - assert(this->size() > 0); - return *begin(); -} -value& value::back() -{ - assert(this->size() > 0); - return *std::prev(end()); -} -const value& value::back() const -{ - assert(this->size() > 0); - return *std::prev(end()); -} -value& value::at(std::size_t i) -{ - auto* a = if_array_impl(x); - if(a == nullptr) - MIGRAPHX_THROW("Not an array"); - return a->at(i); -} -const value& value::at(std::size_t i) const -{ - auto* a = if_array_impl(x); - if(a == nullptr) - MIGRAPHX_THROW("Not an array"); - return a->at(i); -} -value& value::at(const std::string& pkey) -{ - auto* r = find(pkey); - if(r == nullptr) - MIGRAPHX_THROW("Not an object"); - if(r == end()) - MIGRAPHX_THROW("Key not found: " + pkey); - return *r; -} -const value& value::at(const std::string& pkey) const -{ - const auto* r = find(pkey); - if(r == nullptr) - MIGRAPHX_THROW("Not an object for field: " + pkey); - if(r == end()) - MIGRAPHX_THROW("Key not found: " + pkey); - return *r; -} -value& value::operator[](std::size_t i) -{ - assert(i < this->size()); - return *(begin() + i); -} -const value& value::operator[](std::size_t i) const -{ - assert(i < this->size()); - return *(begin() + i); -} -value& value::operator[](const std::string& pkey) { return *emplace(pkey, nullptr).first; } - -void value::clear() { get_array_throw(x).clear(); } -void value::resize(std::size_t n) -{ - if(not is_array()) - MIGRAPHX_THROW("Expected an array."); - get_array_impl(x).resize(n); -} -void value::resize(std::size_t n, const value& v) -{ - if(not is_array()) - MIGRAPHX_THROW("Expected an array."); - get_array_impl(x).resize(n, v); -} - -std::pair value::insert(const value& v) -{ - if(v.key.empty()) - { - if(not x) - x = std::make_shared(); - get_array_impl(x).push_back(v); - assert(this->if_array()); - return std::make_pair(&back(), true); - } - else - { - if(not x) - x = std::make_shared(); - auto p = x->if_object()->emplace(v.key, get_array_impl(x).size()); - if(p.second) - get_array_impl(x).push_back(v); - assert(this->if_object()); - return std::make_pair(&get_array_impl(x)[p.first->second], p.second); - } -} -value* value::insert(const value* pos, const value& v) -{ - assert(v.key.empty()); - if(not x) - x = std::make_shared(); - auto&& a = get_array_impl(x); - auto it = a.insert(a.begin() + (pos - begin()), v); - return std::addressof(*it); -} - -value value::without_key() const -{ - value result = *this; - result.key = ""; - return result; -} - -value value::with_key(const std::string& pkey) const -{ - value result = *this; - result.key = pkey; - return result; -} - -template -const T& compare_decay(const T& x) -{ - return x; -} -int compare_decay(std::nullptr_t) { return 0; } - -template -bool compare(const value& x, const value& y, F f) -{ - bool result = false; - x.visit_value([&](auto&& a) { - y.visit_value([&](auto&& b) { - if constexpr(std::is_same{}) - result = f(std::forward_as_tuple(x.get_key(), compare_decay(a)), - std::forward_as_tuple(y.get_key(), compare_decay(b))); - else - assert(false); // NOLINT - }); - }); - return result; -} - -value::type_t value::get_type() const -{ - if(not x) - return null_type; - return x->get_type(); -} - -bool operator==(const value& x, const value& y) -{ - if(x.get_type() != y.get_type()) - return false; - return compare(x, y, std::equal_to<>{}); -} -bool operator!=(const value& x, const value& y) { return not(x == y); } -bool operator<(const value& x, const value& y) -{ - if(x.get_type() != y.get_type()) - return x.get_type() < y.get_type(); - return compare(x, y, std::less<>{}); -} -bool operator<=(const value& x, const value& y) { return not(x > y); } -bool operator>(const value& x, const value& y) { return y < x; } -bool operator>=(const value& x, const value& y) { return not(x < y); } - -void print_value(std::ostream& os, std::nullptr_t) { os << "null"; } - -template -void print_value(std::ostream& os, const T& x) -{ - os << x; -} - -template -void print_value(std::ostream& os, const std::pair& x) -{ - os << x.first; - os << ": "; - print_value(os, x.second); -} - -void print_value(std::ostream& os, const std::vector& x) -{ - os << "{"; - os << to_string_range(x); - os << "}"; -} - -void print_value(std::ostream& os, const value::binary& x) { os << x; } - -std::ostream& operator<<(std::ostream& os, const value& d) -{ - d.visit([&](auto&& y) { print_value(os, y); }); - return os; -} - -template -std::size_t value_hash(const std::string& key, const T& x) -{ - std::size_t h = hash_value(key); - hash_combine(h, x); - return h; -} - -std::size_t value_hash(const std::string& key, std::nullptr_t) { return hash_value(key); } - -std::size_t value_hash(const std::string& key, const std::vector& x) -{ - std::size_t h = hash_value(key); - for(const auto& v : x) - hash_combine(h, v); - return h; -} -std::size_t value_hash(const std::string& key, const value::binary& x) -{ - std::size_t h = hash_value(key); - for(const auto& v : x) - hash_combine(h, v); - return h; -} - -std::size_t value::hash() const -{ - std::size_t h = 0; - this->visit_value([&](const auto& a) { h = value_hash(this->get_key(), a); }); - return h; -} - -void value::debug_print(bool show_type) const -{ - if(show_type) - { - switch(get_type()) - { -#define MIGRAPHX_VALUE_GENERATE_TYPE_STRING_CASE(vt, cpp_type) \ - case vt##_type: std::cout << #vt << ": "; break; - MIGRAPHX_VISIT_VALUE_TYPES(MIGRAPHX_VALUE_GENERATE_TYPE_STRING_CASE) - MIGRAPHX_VALUE_GENERATE_TYPE_STRING_CASE(null, ) - MIGRAPHX_VALUE_GENERATE_TYPE_STRING_CASE(array, ) - MIGRAPHX_VALUE_GENERATE_TYPE_STRING_CASE(object, ) - } - } - std::cout << *this << std::endl; -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/verify_args.cpp b/docker/rocm/migraphx/verify_args.cpp deleted file mode 100644 index 00b85137b..000000000 --- a/docker/rocm/migraphx/verify_args.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include - -MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_VERIFY_DUMP_DIFF); - -namespace migraphx { -inline namespace MIGRAPHX_INLINE_NS { - -bool verify_args(const std::string& name, - const argument& target_arg, - const verify::expected& ref_arg, - verify::tolerance tols) -{ - bool passed = true; - visit_all(ref_arg.data(), target_arg)([&](auto ref, auto target) { - double rms_error; - passed = - verify::verify_range_with_tolerance(target, verify::expected{ref}, tols, &rms_error); - if(not passed) - { - // TODO: Check for nans - std::cout << "FAILED: " << name << std::endl; - std::cout << "RMS Error: " << rms_error << std::endl; - if(ref.size() < 32 or enabled(MIGRAPHX_VERIFY_DUMP_DIFF{})) - std::cout << "ref:" << ref << std::endl; - if(target.size() < 32 or enabled(MIGRAPHX_VERIFY_DUMP_DIFF{})) - std::cout << "target:" << target << std::endl; - if(verify::range_zero(ref)) - std::cout << "Ref data is all zeros" << std::endl; - if(verify::range_zero(target)) - std::cout << "Target data is all zeros" << std::endl; - - auto mxdiff = verify::max_diff(ref, target); - std::cout << "Max diff: " << mxdiff << std::endl; - - auto idx = verify::mismatch_idx(ref, target, float_equal); - if(idx < verify::range_distance(ref)) - { - std::cout << "Mismatch at " << idx << ": " << ref[idx] << " != " << target[idx] - << std::endl; - } - - auto ref_nan_idx = find_idx(ref, verify::not_finite); - if(ref_nan_idx >= 0) - std::cout << "Non finite number found in ref at " << ref_nan_idx << ": " - << ref[ref_nan_idx] << std::endl; - - auto target_nan_idx = find_idx(target, verify::not_finite); - if(target_nan_idx >= 0) - std::cout << "Non finite number found in target at " << target_nan_idx << ": " - << target[target_nan_idx] << std::endl; - std::cout << std::endl; - } - else - { - if(verify::range_zero(ref)) - std::cout << "Ref data is all zeros" << std::endl; - if(verify::range_zero(target)) - std::cout << "Target data is all zeros" << std::endl; - - auto ref_nan_idx = find_idx(ref, verify::not_finite); - if(ref_nan_idx >= 0) - std::cout << "Non finite number found in ref at " << ref_nan_idx << ": " - << ref[ref_nan_idx] << std::endl; - - auto target_nan_idx = find_idx(target, verify::not_finite); - if(target_nan_idx >= 0) - std::cout << "Non finite number found in target at " << target_nan_idx << ": " - << target[target_nan_idx] << std::endl; - } - }); - return passed; -} - -bool verify_args_with_tolerance(const std::string& name, - const argument& target_arg, - const verify::expected& ref_arg, - std::size_t tolerance) -{ - double rms_tol = 0.001; - target_arg.visit([&](auto ta) { rms_tol = verify::get_rms_tol(ta, tolerance); }); - verify::tolerance tols{rms_tol}; - return verify_args(name, target_arg, ref_arg, tols); -} - -} // namespace MIGRAPHX_INLINE_NS -} // namespace migraphx diff --git a/docker/rocm/migraphx/version.h.in b/docker/rocm/migraphx/version.h.in deleted file mode 100644 index 70ddc38ef..000000000 --- a/docker/rocm/migraphx/version.h.in +++ /dev/null @@ -1,33 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -// clang-format off -#define MIGRAPHX_VERSION_MAJOR @PROJECT_VERSION_MAJOR@ -#define MIGRAPHX_VERSION_MINOR @PROJECT_VERSION_MINOR@ -#define MIGRAPHX_VERSION_PATCH @PROJECT_VERSION_PATCH@ -#define MIGRAPHX_VERSION_TWEAK "@PROJECT_VERSION_TWEAK@" -#define MIGRAPHX_SO_MAJOR_VERSION \ - @PROJECT_VERSION_MAJOR@ * 1000 * 1000 + \ - @PROJECT_VERSION_MINOR@ * 1000 + \ - @PROJECT_VERSION_PATCH@ -// clang-format on