onnx.hpp 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760
  1. // This file is part of OpenCV project.
  2. // It is subject to the license terms in the LICENSE file found in the top-level directory
  3. // of this distribution and at http://opencv.org/license.html.
  4. //
  5. // Copyright (C) 2020-2021 Intel Corporation
  6. #ifndef OPENCV_GAPI_INFER_ONNX_HPP
  7. #define OPENCV_GAPI_INFER_ONNX_HPP
  8. #include <unordered_map>
  9. #include <string>
  10. #include <array>
  11. #include <tuple> // tuple, tuple_size
  12. #include <map>
  13. #include <opencv2/gapi/opencv_includes.hpp>
  14. #include <opencv2/gapi/util/any.hpp>
  15. #include <opencv2/gapi/util/optional.hpp>
  16. #include <opencv2/core/cvdef.h> // GAPI_EXPORTS
  17. #include <opencv2/gapi/gkernel.hpp> // GKernelPackage
  18. #include <opencv2/gapi/infer.hpp> // Generic
  19. namespace cv {
  20. namespace gapi {
  21. /**
  22. * @brief This namespace contains G-API ONNX Runtime backend functions, structures, and symbols.
  23. */
  24. namespace onnx {
  25. /**
  26. * @brief This namespace contains Execution Providers structures for G-API ONNX Runtime backend.
  27. */
  28. namespace ep {
  29. /**
  30. * @brief This structure provides functions
  31. * that fill inference options for ONNX CoreML Execution Provider.
  32. * Please follow https://onnxruntime.ai/docs/execution-providers/CoreML-ExecutionProvider.html#coreml-execution-provider
  33. */
  34. struct GAPI_EXPORTS_W_SIMPLE CoreML {
  35. /** @brief Class constructor.
  36. Constructs CoreML parameters.
  37. */
  38. GAPI_WRAP
  39. CoreML() = default;
  40. /** @brief Limit CoreML Execution Provider to run on CPU only.
  41. This function is used to limit CoreML to run on CPU only.
  42. Please follow: https://onnxruntime.ai/docs/execution-providers/CoreML-ExecutionProvider.html#coreml_flag_use_cpu_only
  43. @return reference to this parameter structure.
  44. */
  45. GAPI_WRAP
  46. CoreML& cfgUseCPUOnly() {
  47. use_cpu_only = true;
  48. return *this;
  49. }
  50. /** @brief Enable CoreML EP to run on a subgraph in the body of a control flow ONNX operator (i.e. a Loop, Scan or If operator).
  51. This function is used to enable CoreML EP to run on
  52. a subgraph of a control flow of ONNX operation.
  53. Please follow: https://onnxruntime.ai/docs/execution-providers/CoreML-ExecutionProvider.html#coreml_flag_enable_on_subgraph
  54. @return reference to this parameter structure.
  55. */
  56. GAPI_WRAP
  57. CoreML& cfgEnableOnSubgraph() {
  58. enable_on_subgraph = true;
  59. return *this;
  60. }
  61. /** @brief Enable CoreML EP to run only on Apple Neural Engine.
  62. This function is used to enable CoreML EP to run only on Apple Neural Engine.
  63. Please follow: https://onnxruntime.ai/docs/execution-providers/CoreML-ExecutionProvider.html#coreml_flag_only_enable_device_with_ane
  64. @return reference to this parameter structure.
  65. */
  66. GAPI_WRAP
  67. CoreML& cfgEnableOnlyNeuralEngine() {
  68. enable_only_ane = true;
  69. return *this;
  70. }
  71. bool use_cpu_only = false;
  72. bool enable_on_subgraph = false;
  73. bool enable_only_ane = false;
  74. };
  75. /**
  76. * @brief This structure provides functions
  77. * that fill inference options for CUDA Execution Provider.
  78. * Please follow https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#cuda-execution-provider
  79. */
  80. struct GAPI_EXPORTS_W_SIMPLE CUDA {
  81. // NB: Used from python.
  82. /// @private -- Exclude this constructor from OpenCV documentation
  83. GAPI_WRAP
  84. CUDA() = default;
  85. /** @brief Class constructor.
  86. Constructs CUDA parameters based on device type information.
  87. @param dev_id Target device id to use.
  88. */
  89. GAPI_WRAP
  90. explicit CUDA(const int dev_id)
  91. : device_id(dev_id) {
  92. }
  93. int device_id;
  94. };
  95. /**
  96. * @brief This structure provides functions
  97. * that fill inference options for TensorRT Execution Provider.
  98. * Please follow https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#tensorrt-execution-provider
  99. */
  100. struct GAPI_EXPORTS_W_SIMPLE TensorRT {
  101. // NB: Used from python.
  102. /// @private -- Exclude this constructor from OpenCV documentation
  103. GAPI_WRAP
  104. TensorRT() = default;
  105. /** @brief Class constructor.
  106. Constructs TensorRT parameters based on device type information.
  107. @param dev_id Target device id to use.
  108. */
  109. GAPI_WRAP
  110. explicit TensorRT(const int dev_id)
  111. : device_id(dev_id) {
  112. }
  113. int device_id;
  114. };
  115. /**
  116. * @brief This structure provides functions
  117. * that fill inference options for ONNX OpenVINO Execution Provider.
  118. * Please follow https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options
  119. */
  120. struct GAPI_EXPORTS_W_SIMPLE OpenVINO {
  121. // NB: Used from python.
  122. /// @private -- Exclude this constructor from OpenCV documentation
  123. GAPI_WRAP
  124. OpenVINO() = default;
  125. /** @brief Class constructor.
  126. Constructs OpenVINO parameters based on device type information.
  127. @param dev_type Target device type to use. ("CPU", "GPU", "GPU.0" etc)
  128. */
  129. GAPI_WRAP
  130. explicit OpenVINO(const std::string &dev_type)
  131. : device_type(dev_type) {
  132. }
  133. /** @brief Class constructor.
  134. Constructs OpenVINO parameters based on map of options passed.
  135. * @param params A map of parameter names and their corresponding string values.
  136. */
  137. GAPI_WRAP
  138. explicit OpenVINO(const std::map<std::string, std::string>& params)
  139. : params_map(params) {
  140. }
  141. /** @brief Specifies OpenVINO Execution Provider cache dir.
  142. This function is used to explicitly specify the path to save and load
  143. the blobs enabling model caching feature.
  144. @param dir Path to the directory what will be used as cache.
  145. @return reference to this parameter structure.
  146. */
  147. GAPI_WRAP
  148. OpenVINO& cfgCacheDir(const std::string &dir) {
  149. if (!params_map.empty()) {
  150. cv::util::throw_error(std::logic_error("ep::OpenVINO cannot be changed if"
  151. "created from the parameters map."));
  152. }
  153. cache_dir = dir;
  154. return *this;
  155. }
  156. /** @brief Specifies OpenVINO Execution Provider number of threads.
  157. This function is used to override the accelerator default value
  158. of number of threads with this value at runtime.
  159. @param nthreads Number of threads.
  160. @return reference to this parameter structure.
  161. */
  162. GAPI_WRAP
  163. OpenVINO& cfgNumThreads(size_t nthreads) {
  164. if (!params_map.empty()) {
  165. cv::util::throw_error(std::logic_error("ep::OpenVINO cannot be changed if"
  166. "created from the parameters map."));
  167. }
  168. num_of_threads = nthreads;
  169. return *this;
  170. }
  171. /** @brief Enables OpenVINO Execution Provider opencl throttling.
  172. This function is used to enable OpenCL queue throttling for GPU devices
  173. (reduces CPU utilization when using GPU).
  174. @return reference to this parameter structure.
  175. */
  176. GAPI_WRAP
  177. OpenVINO& cfgEnableOpenCLThrottling() {
  178. if (!params_map.empty()) {
  179. cv::util::throw_error(std::logic_error("ep::OpenVINO cannot be changed if"
  180. "created from the parameters map."));
  181. }
  182. enable_opencl_throttling = true;
  183. return *this;
  184. }
  185. /** @brief Enables OpenVINO Execution Provider dynamic shapes.
  186. This function is used to enable OpenCL queue throttling for GPU devices
  187. (reduces CPU utilization when using GPU).
  188. This function is used to enable work with dynamic shaped models
  189. whose shape will be set dynamically based on the infer input
  190. image/data shape at run time in CPU.
  191. @return reference to this parameter structure.
  192. */
  193. GAPI_WRAP
  194. OpenVINO& cfgEnableDynamicShapes() {
  195. if (!params_map.empty()) {
  196. cv::util::throw_error(std::logic_error("ep::OpenVINO cannot be changed if"
  197. "created from the parameters map."));
  198. }
  199. enable_dynamic_shapes = true;
  200. return *this;
  201. }
  202. std::string device_type;
  203. std::string cache_dir;
  204. size_t num_of_threads = 0;
  205. bool enable_opencl_throttling = false;
  206. bool enable_dynamic_shapes = false;
  207. std::map<std::string, std::string> params_map;
  208. };
  209. /**
  210. * @brief This structure provides functions
  211. * that fill inference options for ONNX DirectML Execution Provider.
  212. * Please follow https://onnxruntime.ai/docs/execution-providers/DirectML-ExecutionProvider.html#directml-execution-provider
  213. */
  214. class GAPI_EXPORTS_W_SIMPLE DirectML {
  215. public:
  216. // NB: Used from python.
  217. /// @private -- Exclude this constructor from OpenCV documentation
  218. GAPI_WRAP
  219. DirectML() = default;
  220. /** @brief Class constructor.
  221. Constructs DirectML parameters based on device id.
  222. @param device_id Target device id to use. ("0", "1", etc)
  223. */
  224. GAPI_WRAP
  225. explicit DirectML(const int device_id) : ddesc(device_id) { };
  226. /** @brief Class constructor.
  227. Constructs DirectML parameters based on adapter name.
  228. @param adapter_name Target adapter_name to use.
  229. */
  230. GAPI_WRAP
  231. explicit DirectML(const std::string &adapter_name) : ddesc(adapter_name) { };
  232. using DeviceDesc = cv::util::variant<int, std::string>;
  233. DeviceDesc ddesc;
  234. };
  235. using EP = cv::util::variant< cv::util::monostate
  236. , OpenVINO
  237. , DirectML
  238. , CoreML
  239. , CUDA
  240. , TensorRT>;
  241. } // namespace ep
  242. GAPI_EXPORTS cv::gapi::GBackend backend();
  243. enum class TraitAs: int {
  244. TENSOR, //!< G-API traits an associated cv::Mat as a raw tensor
  245. // and passes dimensions as-is
  246. IMAGE //!< G-API traits an associated cv::Mat as an image so
  247. // creates an "image" blob (NCHW/NHWC, etc)
  248. };
  249. using PostProc = std::function<void(const std::unordered_map<std::string, cv::Mat> &,
  250. std::unordered_map<std::string, cv::Mat> &)>;
  251. namespace detail {
  252. /**
  253. * @brief This structure contains description of inference parameters
  254. * which is specific to ONNX models.
  255. */
  256. struct ParamDesc {
  257. std::string model_path; //!< Path to model.
  258. // NB: nun_* may differ from topology's real input/output port numbers
  259. // (e.g. topology's partial execution)
  260. std::size_t num_in; //!< How many inputs are defined in the operation
  261. std::size_t num_out; //!< How many outputs are defined in the operation
  262. // NB: Here order follows the `Net` API
  263. std::vector<std::string> input_names; //!< Names of input network layers.
  264. std::vector<std::string> output_names; //!< Names of output network layers.
  265. using ConstInput = std::pair<cv::Mat, TraitAs>;
  266. std::unordered_map<std::string, ConstInput> const_inputs; //!< Map with pair of name of network layer and ConstInput which will be associated with this.
  267. std::vector<cv::Scalar> mean; //!< Mean values for preprocessing.
  268. std::vector<cv::Scalar> stdev; //!< Standard deviation values for preprocessing.
  269. std::vector<cv::GMatDesc> out_metas; //!< Out meta information about your output (type, dimension).
  270. PostProc custom_post_proc; //!< Post processing function.
  271. std::vector<bool> normalize; //!< Vector of bool values that enabled or disabled normalize of input data.
  272. std::vector<std::string> names_to_remap; //!< Names of output layers that will be processed in PostProc function.
  273. bool is_generic;
  274. // TODO: Needs to modify the rest of ParamDesc accordingly to support
  275. // both generic and non-generic options without duplication
  276. // (as it was done for the OV IE backend)
  277. // These values are pushed into the respective vector<> fields above
  278. // when the generic infer parameters are unpacked (see GONNXBackendImpl::unpackKernel)
  279. std::unordered_map<std::string, std::pair<cv::Scalar, cv::Scalar> > generic_mstd;
  280. std::unordered_map<std::string, bool> generic_norm;
  281. std::map<std::string, std::string> session_options;
  282. std::vector<cv::gapi::onnx::ep::EP> execution_providers;
  283. bool disable_mem_pattern;
  284. cv::util::optional<int> opt_level;
  285. };
  286. } // namespace detail
  287. template<typename Net>
  288. struct PortCfg {
  289. using In = std::array
  290. < std::string
  291. , std::tuple_size<typename Net::InArgs>::value >;
  292. using Out = std::array
  293. < std::string
  294. , std::tuple_size<typename Net::OutArgs>::value >;
  295. using NormCoefs = std::array
  296. < cv::Scalar
  297. , std::tuple_size<typename Net::InArgs>::value >;
  298. using Normalize = std::array
  299. < bool
  300. , std::tuple_size<typename Net::InArgs>::value >;
  301. };
  302. /**
  303. * Contains description of inference parameters and kit of functions that
  304. * fill this parameters.
  305. */
  306. template<typename Net> class Params {
  307. public:
  308. /** @brief Class constructor.
  309. Constructs Params based on model information and sets default values for other
  310. inference description parameters.
  311. @param model Path to model (.onnx file).
  312. */
  313. Params(const std::string &model) {
  314. desc.model_path = model;
  315. desc.num_in = std::tuple_size<typename Net::InArgs>::value;
  316. desc.num_out = std::tuple_size<typename Net::OutArgs>::value;
  317. desc.is_generic = false;
  318. desc.disable_mem_pattern = false;
  319. }
  320. /** @brief Specifies sequence of network input layers names for inference.
  321. The function is used to associate data of graph inputs with input layers of
  322. network topology. Number of names has to match the number of network inputs. If a network
  323. has only one input layer, there is no need to call it as the layer is
  324. associated with input automatically but this doesn't prevent you from
  325. doing it yourself. Count of names has to match to number of network inputs.
  326. @param layer_names std::array<std::string, N> where N is the number of inputs
  327. as defined in the @ref G_API_NET. Contains names of input layers.
  328. @return the reference on modified object.
  329. */
  330. Params<Net>& cfgInputLayers(const typename PortCfg<Net>::In &layer_names) {
  331. desc.input_names.assign(layer_names.begin(), layer_names.end());
  332. return *this;
  333. }
  334. /** @brief Specifies sequence of output layers names for inference.
  335. The function is used to associate data of graph outputs with output layers of
  336. network topology. If a network has only one output layer, there is no need to call it
  337. as the layer is associated with output automatically but this doesn't prevent
  338. you from doing it yourself. Count of names has to match to number of network
  339. outputs or you can set your own output but for this case you have to
  340. additionally use @ref cfgPostProc function.
  341. @param layer_names std::array<std::string, N> where N is the number of outputs
  342. as defined in the @ref G_API_NET. Contains names of output layers.
  343. @return the reference on modified object.
  344. */
  345. Params<Net>& cfgOutputLayers(const typename PortCfg<Net>::Out &layer_names) {
  346. desc.output_names.assign(layer_names.begin(), layer_names.end());
  347. return *this;
  348. }
  349. /** @brief Sets a constant input.
  350. The function is used to set constant input. This input has to be
  351. a prepared tensor since preprocessing is disabled for this case. You should
  352. provide name of network layer which will receive provided data.
  353. @param layer_name Name of network layer.
  354. @param data cv::Mat that contains data which will be associated with network layer.
  355. @param hint Type of input (TENSOR).
  356. @return the reference on modified object.
  357. */
  358. Params<Net>& constInput(const std::string &layer_name,
  359. const cv::Mat &data,
  360. TraitAs hint = TraitAs::TENSOR) {
  361. desc.const_inputs[layer_name] = {data, hint};
  362. return *this;
  363. }
  364. /** @brief Specifies mean value and standard deviation for preprocessing.
  365. The function is used to set mean value and standard deviation for preprocessing
  366. of input data.
  367. @param m std::array<cv::Scalar, N> where N is the number of inputs
  368. as defined in the @ref G_API_NET. Contains mean values.
  369. @param s std::array<cv::Scalar, N> where N is the number of inputs
  370. as defined in the @ref G_API_NET. Contains standard deviation values.
  371. @return the reference on modified object.
  372. */
  373. Params<Net>& cfgMeanStd(const typename PortCfg<Net>::NormCoefs &m,
  374. const typename PortCfg<Net>::NormCoefs &s) {
  375. desc.mean.assign(m.begin(), m.end());
  376. desc.stdev.assign(s.begin(), s.end());
  377. return *this;
  378. }
  379. /** @brief Configures graph output and provides the post processing function from user.
  380. The function is used when you work with networks with dynamic outputs.
  381. Since we can't know dimensions of inference result needs provide them for
  382. construction of graph output. This dimensions can differ from inference result.
  383. So you have to provide @ref PostProc function that gets information from inference
  384. result and fill output which is constructed by dimensions from out_metas.
  385. @param out_metas Out meta information about your output (type, dimension).
  386. @param remap_function Post processing function, which has two parameters. First is onnx
  387. result, second is graph output. Both parameters is std::map that contain pair of
  388. layer's name and cv::Mat.
  389. @return the reference on modified object.
  390. */
  391. Params<Net>& cfgPostProc(const std::vector<cv::GMatDesc> &out_metas,
  392. const PostProc &remap_function) {
  393. desc.out_metas = out_metas;
  394. desc.custom_post_proc = remap_function;
  395. return *this;
  396. }
  397. /** @overload
  398. Function with a rvalue parameters.
  399. @param out_metas rvalue out meta information about your output (type, dimension).
  400. @param remap_function rvalue post processing function, which has two parameters. First is onnx
  401. result, second is graph output. Both parameters is std::map that contain pair of
  402. layer's name and cv::Mat.
  403. @return the reference on modified object.
  404. */
  405. Params<Net>& cfgPostProc(std::vector<cv::GMatDesc> &&out_metas,
  406. PostProc &&remap_function) {
  407. desc.out_metas = std::move(out_metas);
  408. desc.custom_post_proc = std::move(remap_function);
  409. return *this;
  410. }
  411. /** @overload
  412. The function has additional parameter names_to_remap. This parameter provides
  413. information about output layers which will be used for inference and post
  414. processing function.
  415. @param out_metas Out meta information.
  416. @param remap_function Post processing function.
  417. @param names_to_remap Names of output layers. network's inference will
  418. be done on these layers. Inference's result will be processed in post processing
  419. function using these names.
  420. @return the reference on modified object.
  421. */
  422. Params<Net>& cfgPostProc(const std::vector<cv::GMatDesc> &out_metas,
  423. const PostProc &remap_function,
  424. const std::vector<std::string> &names_to_remap) {
  425. desc.out_metas = out_metas;
  426. desc.custom_post_proc = remap_function;
  427. desc.names_to_remap = names_to_remap;
  428. return *this;
  429. }
  430. /** @overload
  431. Function with a rvalue parameters and additional parameter names_to_remap.
  432. @param out_metas rvalue out meta information.
  433. @param remap_function rvalue post processing function.
  434. @param names_to_remap rvalue names of output layers. network's inference will
  435. be done on these layers. Inference's result will be processed in post processing
  436. function using these names.
  437. @return the reference on modified object.
  438. */
  439. Params<Net>& cfgPostProc(std::vector<cv::GMatDesc> &&out_metas,
  440. PostProc &&remap_function,
  441. std::vector<std::string> &&names_to_remap) {
  442. desc.out_metas = std::move(out_metas);
  443. desc.custom_post_proc = std::move(remap_function);
  444. desc.names_to_remap = std::move(names_to_remap);
  445. return *this;
  446. }
  447. /** @brief Specifies normalize parameter for preprocessing.
  448. The function is used to set normalize parameter for preprocessing of input data.
  449. @param normalizations std::array<cv::Scalar, N> where N is the number of inputs
  450. as defined in the @ref G_API_NET. Сontains bool values that enabled or disabled
  451. normalize of input data.
  452. @return the reference on modified object.
  453. */
  454. Params<Net>& cfgNormalize(const typename PortCfg<Net>::Normalize &normalizations) {
  455. desc.normalize.assign(normalizations.begin(), normalizations.end());
  456. return *this;
  457. }
  458. /** @brief Adds execution provider for runtime.
  459. The function is used to add ONNX Runtime OpenVINO Execution Provider options.
  460. @param ep OpenVINO Execution Provider options.
  461. @see cv::gapi::onnx::ep::OpenVINO.
  462. @return the reference on modified object.
  463. */
  464. Params<Net>& cfgAddExecutionProvider(ep::OpenVINO&& ep) {
  465. desc.execution_providers.emplace_back(std::move(ep));
  466. return *this;
  467. }
  468. /** @brief Adds execution provider for runtime.
  469. The function is used to add ONNX Runtime DirectML Execution Provider options.
  470. @param ep DirectML Execution Provider options.
  471. @see cv::gapi::onnx::ep::DirectML.
  472. @return the reference on modified object.
  473. */
  474. Params<Net>& cfgAddExecutionProvider(ep::DirectML&& ep) {
  475. desc.execution_providers.emplace_back(std::move(ep));
  476. return *this;
  477. }
  478. /** @brief Adds execution provider for runtime.
  479. The function is used to add ONNX Runtime CoreML Execution Provider options.
  480. @param ep CoreML Execution Provider options.
  481. @see cv::gapi::onnx::ep::CoreML.
  482. @return the reference on modified object.
  483. */
  484. Params<Net>& cfgAddExecutionProvider(ep::CoreML&& ep) {
  485. desc.execution_providers.emplace_back(std::move(ep));
  486. return *this;
  487. }
  488. /** @brief Adds execution provider for runtime.
  489. The function is used to add ONNX Runtime CUDA Execution Provider options.
  490. @param ep CUDA Execution Provider options.
  491. @see cv::gapi::onnx::ep::CUDA.
  492. @return the reference on modified object.
  493. */
  494. Params<Net>& cfgAddExecutionProvider(ep::CUDA&& ep) {
  495. desc.execution_providers.emplace_back(std::move(ep));
  496. return *this;
  497. }
  498. /** @brief Adds execution provider for runtime.
  499. The function is used to add ONNX Runtime TensorRT Execution Provider options.
  500. @param ep TensorRT Execution Provider options.
  501. @see cv::gapi::onnx::ep::TensorRT.
  502. @return the reference on modified object.
  503. */
  504. Params<Net>& cfgAddExecutionProvider(ep::TensorRT&& ep) {
  505. desc.execution_providers.emplace_back(std::move(ep));
  506. return *this;
  507. }
  508. /** @brief Disables the memory pattern optimization.
  509. @return the reference on modified object.
  510. */
  511. Params<Net>& cfgDisableMemPattern() {
  512. desc.disable_mem_pattern = true;
  513. return *this;
  514. }
  515. /** @brief Configures session options for ONNX Runtime.
  516. This function is used to set various session options for the ONNX Runtime
  517. session by accepting a map of key-value pairs.
  518. @param options A map of session option to be applied to the ONNX Runtime session.
  519. @return the reference on modified object.
  520. */
  521. Params<Net>& cfgSessionOptions(const std::map<std::string, std::string>& options) {
  522. desc.session_options.insert(options.begin(), options.end());
  523. return *this;
  524. }
  525. /** @brief Configures optimization level for ONNX Runtime.
  526. @param opt_level [optimization level]: Valid values are 0 (disable), 1 (basic), 2 (extended), 99 (all).
  527. Please see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels.
  528. @return the reference on modified object.
  529. */
  530. Params<Net>& cfgOptLevel(const int opt_level) {
  531. desc.opt_level = cv::util::make_optional(opt_level);
  532. return *this;
  533. }
  534. // BEGIN(G-API's network parametrization API)
  535. GBackend backend() const { return cv::gapi::onnx::backend(); }
  536. std::string tag() const { return Net::tag(); }
  537. cv::util::any params() const { return { desc }; }
  538. // END(G-API's network parametrization API)
  539. protected:
  540. detail::ParamDesc desc;
  541. };
  542. /*
  543. * @brief This structure provides functions for generic network type that
  544. * fill inference parameters.
  545. * @see struct Generic
  546. */
  547. template<>
  548. class Params<cv::gapi::Generic> {
  549. public:
  550. /** @brief Class constructor.
  551. Constructs Params based on input information and sets default values for other
  552. inference description parameters.
  553. @param tag string tag of the network for which these parameters are intended.
  554. @param model_path path to model file (.onnx file).
  555. */
  556. Params(const std::string& tag, const std::string& model_path)
  557. : desc{ model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, {}, false, {} }, m_tag(tag) {}
  558. /** @see onnx::Params::cfgMeanStdDev. */
  559. void cfgMeanStdDev(const std::string &layer,
  560. const cv::Scalar &m,
  561. const cv::Scalar &s) {
  562. desc.generic_mstd[layer] = std::make_pair(m, s);
  563. }
  564. /** @see onnx::Params::cfgNormalize. */
  565. void cfgNormalize(const std::string &layer, bool flag) {
  566. desc.generic_norm[layer] = flag;
  567. }
  568. /** @see onnx::Params::cfgAddExecutionProvider. */
  569. void cfgAddExecutionProvider(ep::OpenVINO&& ep) {
  570. desc.execution_providers.emplace_back(std::move(ep));
  571. }
  572. /** @see onnx::Params::cfgAddExecutionProvider. */
  573. void cfgAddExecutionProvider(ep::DirectML&& ep) {
  574. desc.execution_providers.emplace_back(std::move(ep));
  575. }
  576. /** @see onnx::Params::cfgAddExecutionProvider. */
  577. void cfgAddExecutionProvider(ep::CoreML&& ep) {
  578. desc.execution_providers.emplace_back(std::move(ep));
  579. }
  580. /** @see onnx::Params::cfgAddExecutionProvider. */
  581. void cfgAddExecutionProvider(ep::CUDA&& ep) {
  582. desc.execution_providers.emplace_back(std::move(ep));
  583. }
  584. /** @see onnx::Params::cfgAddExecutionProvider. */
  585. void cfgAddExecutionProvider(ep::TensorRT&& ep) {
  586. desc.execution_providers.emplace_back(std::move(ep));
  587. }
  588. /** @see onnx::Params::cfgDisableMemPattern. */
  589. void cfgDisableMemPattern() {
  590. desc.disable_mem_pattern = true;
  591. }
  592. /** @see onnx::Params::cfgSessionOptions. */
  593. void cfgSessionOptions(const std::map<std::string, std::string>& options) {
  594. desc.session_options.insert(options.begin(), options.end());
  595. }
  596. /** @see onnx::Params::cfgOptLevel. */
  597. void cfgOptLevel(const int opt_level) {
  598. desc.opt_level = cv::util::make_optional(opt_level);
  599. }
  600. // BEGIN(G-API's network parametrization API)
  601. GBackend backend() const { return cv::gapi::onnx::backend(); }
  602. std::string tag() const { return m_tag; }
  603. cv::util::any params() const { return { desc }; }
  604. // END(G-API's network parametrization API)
  605. protected:
  606. detail::ParamDesc desc;
  607. std::string m_tag;
  608. };
  609. } // namespace onnx
  610. } // namespace gapi
  611. } // namespace cv
  612. #endif // OPENCV_GAPI_INFER_HPP