How to use the onnxruntime.core.providers.nuphar.scripts.symbolic_shape_infer.SymbolicShapeInference function in onnxruntime

To help you get started, we’ve selected a few onnxruntime examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github microsoft / onnxruntime / onnxruntime / core / providers / nuphar / scripts / symbolic_shape_infer.py View on Github external
def infer_shapes(input_model, output_model, int_max=2**31 - 1, auto_merge=False, guess_output_rank=False, verbose=0):
        in_mp = onnx.load(input_model)
        if get_opset(in_mp) < 7:
            print('Only support models of opset 7 and above.')
            return
        symbolic_shape_inference = SymbolicShapeInference(int_max, auto_merge, guess_output_rank, verbose)
        all_shapes_inferred = False
        symbolic_shape_inference._preprocess(in_mp)
        while symbolic_shape_inference.run_:
            all_shapes_inferred = symbolic_shape_inference._infer_impl(in_mp)
        symbolic_shape_inference._update_output_from_vi()
        if output_model:
            onnx.save(symbolic_shape_inference.out_mp_, output_model)
        if not all_shapes_inferred:
            sys.exit(1)
github microsoft / onnxruntime / onnxruntime / core / providers / nuphar / scripts / rnn_benchmark.py View on Github external
# there are some local thread pool inside LSTM/GRU CPU kernel
    # that cannot be controlled by OMP or intra_op_num_threads
    sess = onnxruntime.InferenceSession(model_name, providers=['CPUExecutionProvider'])
    count, duration, per_iter_cost = perf_run(sess, feeds, min_counts=top_n, min_duration_seconds=min_duration_seconds)
    avg_rnn = top_n_avg(per_iter_cost, top_n)
    print('perf_rnn (with default threads) {}: run for {} iterations, top {} avg {:.3f} ms'.format(model_name, count, top_n, avg_rnn))

    # run converted model in Nuphar, using specified threads
    with ScopedSetNumThreads(num_threads) as scoped_set_num_threads:
        # run Scan model converted from original in Nuphar
        from .model_editor import convert_to_scan_model
        from .symbolic_shape_infer import SymbolicShapeInference
        scan_model_name = os.path.splitext(model_name)[0] + '_scan.onnx'
        convert_to_scan_model(model_name, scan_model_name)
        # note that symbolic shape inference is needed because model has symbolic batch dim, thus init_state is ConstantOfShape
        SymbolicShapeInference.infer_shapes(scan_model_name, scan_model_name)
        sess = onnxruntime.InferenceSession(scan_model_name)
        count, duration, per_iter_cost = perf_run(sess, feeds, min_counts=top_n, min_duration_seconds=min_duration_seconds)
        avg_scan = top_n_avg(per_iter_cost, top_n)
        print('perf_scan (with {} threads) {}: run for {} iterations, top {} avg {:.3f} ms'.format(num_threads, scan_model_name, count, top_n, avg_scan))

        # quantize Scan model to int8 and run in Nuphar
        from .model_quantizer import convert_matmul_model
        int8_model_name = os.path.splitext(model_name)[0] + '_int8.onnx'
        convert_matmul_model(scan_model_name, int8_model_name)
        SymbolicShapeInference.infer_shapes(int8_model_name, int8_model_name)
        sess = onnxruntime.InferenceSession(int8_model_name)
        count, duration, per_iter_cost = perf_run(sess, feeds, min_counts=top_n, min_duration_seconds=min_duration_seconds)
        avg_int8 = top_n_avg(per_iter_cost, top_n)
        print('perf_int8 (with {} threads) {}: run for {} iterations, top {} avg {:.3f} ms'.format(num_threads, int8_model_name, count, top_n, avg_int8))

    return avg_rnn, avg_scan, avg_int8
github microsoft / onnxruntime / onnxruntime / core / providers / nuphar / scripts / rnn_benchmark.py View on Github external
from .model_editor import convert_to_scan_model
        from .symbolic_shape_infer import SymbolicShapeInference
        scan_model_name = os.path.splitext(model_name)[0] + '_scan.onnx'
        convert_to_scan_model(model_name, scan_model_name)
        # note that symbolic shape inference is needed because model has symbolic batch dim, thus init_state is ConstantOfShape
        SymbolicShapeInference.infer_shapes(scan_model_name, scan_model_name)
        sess = onnxruntime.InferenceSession(scan_model_name)
        count, duration, per_iter_cost = perf_run(sess, feeds, min_counts=top_n, min_duration_seconds=min_duration_seconds)
        avg_scan = top_n_avg(per_iter_cost, top_n)
        print('perf_scan (with {} threads) {}: run for {} iterations, top {} avg {:.3f} ms'.format(num_threads, scan_model_name, count, top_n, avg_scan))

        # quantize Scan model to int8 and run in Nuphar
        from .model_quantizer import convert_matmul_model
        int8_model_name = os.path.splitext(model_name)[0] + '_int8.onnx'
        convert_matmul_model(scan_model_name, int8_model_name)
        SymbolicShapeInference.infer_shapes(int8_model_name, int8_model_name)
        sess = onnxruntime.InferenceSession(int8_model_name)
        count, duration, per_iter_cost = perf_run(sess, feeds, min_counts=top_n, min_duration_seconds=min_duration_seconds)
        avg_int8 = top_n_avg(per_iter_cost, top_n)
        print('perf_int8 (with {} threads) {}: run for {} iterations, top {} avg {:.3f} ms'.format(num_threads, int8_model_name, count, top_n, avg_int8))

    return avg_rnn, avg_scan, avg_int8
github microsoft / onnxruntime / onnxruntime / core / providers / nuphar / scripts / symbolic_shape_infer.py View on Github external
parser = argparse.ArgumentParser()
  parser.add_argument('--input', required=True, help='The input model file')
  parser.add_argument('--output', help='The input model file')
  parser.add_argument('--auto_merge', help='Automatically merge symbolic dims when confliction happens', action='store_true', default=False)
  parser.add_argument('--int_max', help='maximum value for integer to be treated as boundless for ops like slice', type=int, default=2**31 - 1)
  parser.add_argument('--guess_output_rank', help='guess output rank to be the same as input 0 for unknown ops', action='store_true', default=False)
  parser.add_argument('--verbose', help='Prints detailed logs of inference, 0: turn off, 1: warnings, 3: detailed', type=int, default=0)
  return parser.parse_args()

if __name__ == '__main__':
    args = parse_arguments()
    print('input model: ' + args.input)
    if args.output:
        print('output model ' + args.output)
    print('Doing symbolic shape inference...')
    out_mp = SymbolicShapeInference.infer_shapes(args.input, args.output, args.int_max, args.auto_merge, args.guess_output_rank, args.verbose)
    print('Done!')
github microsoft / onnxruntime / onnxruntime / core / providers / nuphar / scripts / model_editor.py View on Github external
args = parse_arguments()
    print('input model: ' + args.input)
    print('output model ' + args.output)
    if args.mode == 'to_scan':
        print('Convert LSTM/GRU/RNN to Scan...')
        convert_to_scan_model(args.input, args.output)
    elif args.mode == 'opt_inproj':
        print('Optimize input projection in Scan...')
        optimize_input_projection(args.input, args.output)
    elif args.mode == 'remove_initializers_from_inputs':
        print('Remove all initializers from input for model with IR version >= 4...')
        remove_initializers_from_inputs(args.input, args.output)
    else:
        raise NotImplementedError('Unknown mode')
    print('Running symbolic shape inference on output model')
    SymbolicShapeInference.infer_shapes(args.output, args.output, auto_merge=True)
    print('Done!')
github microsoft / onnxruntime / onnxruntime / core / providers / nuphar / scripts / symbolic_shape_infer.py View on Github external
print('Inferencing subgraph of node {} with output({}...): {}'.format(node.name, node.output[0], node.op_type))
        # node inputs are not passed directly to the subgraph
        # it's up to the node dispatcher to prepare subgraph input
        # for example, with Scan/Loop, subgraph input shape would be trimmed from node input shape
        # besides, inputs in subgraph could shadow implicit inputs
        subgraph_inputs = set([i.name for i in list(subgraph.initializer) + list(subgraph.input)])
        subgraph_implicit_input = set([name for name in self.known_vi_.keys() if not name in subgraph_inputs])
        tmp_graph = helper.make_graph(list(subgraph.node),
                                      'tmp',
                                      list(subgraph.input) + [self.known_vi_[i] for i in subgraph_implicit_input],
                                      [helper.make_tensor_value_info(i.name, onnx.TensorProto.UNDEFINED, None) for i in subgraph.output])
        tmp_graph.initializer.extend([i for i in self.out_mp_.graph.initializer if i.name in subgraph_implicit_input])
        tmp_graph.initializer.extend(subgraph.initializer)
        self.tmp_mp_.graph.CopyFrom(tmp_graph)

        symbolic_shape_inference = SymbolicShapeInference(self.int_max_, self.auto_merge_, self.guess_output_rank_, self.verbose_)
        all_shapes_inferred = False
        symbolic_shape_inference._preprocess(self.tmp_mp_)
        symbolic_shape_inference.suggested_merge_ = self.suggested_merge_.copy()
        while symbolic_shape_inference.run_:
            all_shapes_inferred = symbolic_shape_inference._infer_impl(self.tmp_mp_, self.sympy_data_.copy())
        symbolic_shape_inference._update_output_from_vi()
        if use_node_input:
            # if subgraph uses node input, it needs to update to merged dims
            subgraph.ClearField('input')
            subgraph.input.extend(symbolic_shape_inference.out_mp_.graph.input[:len(node.input)])
        subgraph.ClearField('output')
        subgraph.output.extend(symbolic_shape_inference.out_mp_.graph.output)
        subgraph.ClearField('value_info')
        subgraph.value_info.extend(symbolic_shape_inference.out_mp_.graph.value_info)
        subgraph.ClearField('node')
        subgraph.node.extend(symbolic_shape_inference.out_mp_.graph.node)