Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Connect ONNX's output and CoreML's output via a reshape operator
apply_reshape(scope, rnn_h_name, operator.outputs[1].full_name, container, desired_shape=[1, hidden_size])
else:
# According to CoreML, its two outputs are always identical, so we just need to compute one of them and produce
# the other one using an identity operator. Note that the first ONNX RNN output is undefined in this case.
# Reshape last hidden state's ONNX format to its CoreML format
apply_reshape(scope, rnn_h_name, operator.outputs[0].full_name, container, desired_shape=[1, hidden_size])
if len(operator.outputs) == 2:
# Copy the first output to the second output
container.add_node('Identity', operator.outputs[0].full_name, operator.outputs[1].full_name,
name=scope.get_unique_operator_name('Identity'))
register_converter('simpleRecurrent', convert_simple_rnn)
classifier_op.is_evaluated = True
single_feature_tensor = scope.get_unique_variable_name('single_feature_tensor')
container.add_node('ArrayFeatureExtractor', [classifier_probability_output.full_name, index_tensor],
single_feature_tensor,
op_domain='ai.onnx.ml',
name=scope.get_unique_operator_name('ArrayFeatureExtractor'))
classifier_output_names.append(single_feature_tensor)
concatenated_probabilities = scope.get_unique_variable_name('concatenated_predictions_tensor')
apply_concat(scope, classifier_output_names, concatenated_probabilities, container, axis=1)
# to get Prediction from probability
apply_argmax(scope, concatenated_probabilities, operator.outputs[0].full_name, container,
axis=1, keepdims=1)
register_converter('pyspark.ml.classification.OneVsRestModel', convert_one_vs_rest)
def calculate_one_vs_rest_output_shapes(operator):
check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
check_input_and_output_types(operator, good_input_types=[FloatTensorType, Int64TensorType])
if len(operator.inputs[0].type.shape) != 2:
raise RuntimeError('Input must be a [N, C]-tensor')
N = operator.inputs[0].type.shape[0]
operator.outputs[0].type = Int64TensorType(shape=[N])
register_shape_calculator('pyspark.ml.classification.OneVsRestModel',
calculate_one_vs_rest_output_shapes)
def convert_keras_separable_conv1d(scope, operator, container):
is_transpose, n_dims, input_perm, output_perm, weight_perm = get_converter_config(1, False)
convert_keras_conv_core(scope, operator, container, is_transpose, n_dims, input_perm, output_perm, weight_perm)
def convert_keras_separable_conv2d(scope, operator, container):
is_transpose, n_dims, input_perm, output_perm, weight_perm = get_converter_config(2, False)
convert_keras_conv_core(scope, operator, container, is_transpose, n_dims, input_perm, output_perm, weight_perm)
register_converter(Conv1D, convert_keras_conv1d)
register_converter(Conv2D, convert_keras_conv2d)
register_converter(Conv3D, convert_keras_conv3d)
register_converter(Conv2DTranspose, convert_keras_conv_transpose_2d)
register_converter(Conv3DTranspose, convert_keras_conv_transpose_3d)
if StrictVersion(keras.__version__) >= StrictVersion('2.1.5'):
register_converter(DepthwiseConv2D, convert_keras_depthwise_conv_2d)
register_converter(SeparableConv2D, convert_keras_separable_conv2d)
if StrictVersion(keras.__version__) >= StrictVersion('2.1.3'):
register_converter(SeparableConv1D, convert_keras_separable_conv1d)
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
import keras
from ...common._apply_operation import apply_transpose
from ...common._registration import register_converter
def convert_keras_permute(scope, operator, container):
axes = [0] + list(operator.raw_operator.dims)
apply_transpose(scope, operator.inputs[0].full_name, operator.outputs[0].full_name, container, perm=axes)
register_converter(keras.layers.Permute, convert_keras_permute)
# random forest calculate the final score by averaging over all trees'
# outcomes, so all trees' weights are identical.
tree_weight = 1. / op.getNumTrees
for tree_id in range(0, op.getNumTrees):
tree_model = op.trees[tree_id]
tree_df = save_read_sparkml_model_data(operator.raw_params['SparkSession'], tree_model)
tree = sparkml_tree_dataset_to_sklearn(tree_df, is_classifier=False)
add_tree_to_attribute_pairs(attrs, False, tree, tree_id,
tree_weight, 0, False)
container.add_node(op_type, operator.input_full_names, operator.output_full_names[0],
op_domain='ai.onnx.ml', **attrs)
register_converter('pyspark.ml.regression.RandomForestRegressionModel', convert_random_forest_regressor)
register_shape_calculator('pyspark.ml.regression.RandomForestRegressionModel',
calculate_decision_tree_regressor_output_shapes)
from ...common.utils import check_input_and_output_numbers, check_input_and_output_types
from ...common.data_types import *
def convert_chi_sq_selector(scope, operator, container):
op = operator.raw_operator
indices = op.selectedFeatures
indices_tensor = 'indices_tensor'
container.add_initializer(indices_tensor, onnx_proto.TensorProto.INT64, [len(indices)], indices)
container.add_node('ArrayFeatureExtractor',
[operator.input_full_names[0], indices_tensor], operator.output_full_names,
op_domain='ai.onnx.ml',
name=scope.get_unique_operator_name('ArrayFeatureExtractor'))
register_converter('pyspark.ml.feature.ChiSqSelectorModel', convert_chi_sq_selector)
def calculate_chi_sq_selector_shapes(operator):
check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
check_input_and_output_types(operator,
good_input_types=[FloatTensorType, Int64TensorType, StringTensorType])
operator.outputs[0].type = copy.deepcopy(operator.inputs[0].type)
operator.outputs[0].type.shape[1] = len(operator.raw_operator.selectedFeatures)
register_shape_calculator('pyspark.ml.feature.ChiSqSelectorModel', calculate_chi_sq_selector_shapes)
def convert_libsvm(scope, operator, container):
inputs = operator.inputs
model = operator.raw_operator
converter = AnyLibSvmConverter
onnx_nodes = []
outputs = None
converter.validate(model)
converter.convert(operator, scope, container, model, inputs)
# Register the class for processing
register_converter("LibSvmSVC", convert_libsvm)
register_converter("LibSvmSVR", convert_libsvm)
def convert_one_hot_encoder(scope, operator, container):
op_type = 'OneHotEncoder'
attrs = {'name': operator.full_name}
raw_model = operator.raw_operator.oneHotEncoder
if raw_model.HasField('int64Categories'):
attrs['cats_int64s'] = list(int(i) for i in raw_model.int64Categories.vector)
if raw_model.HasField('stringCategories'):
attrs['cats_strings'] = list(str(s).encode('utf-8') for s in raw_model.stringCategories.vector)
container.add_node(op_type, [operator.inputs[0].full_name], [operator.outputs[0].full_name],
op_domain='ai.onnx.ml', **attrs)
register_converter('oneHotEncoder', convert_one_hot_encoder)
def convert_keras_upsample_1d(scope, operator, container):
convert_keras_upsample(scope, operator, container, n_dims=1)
def convert_keras_upsample_2d(scope, operator, container):
convert_keras_upsample(scope, operator, container, n_dims=2)
def convert_keras_upsample_3d(scope, operator, container):
convert_keras_upsample(scope, operator, container, n_dims=3)
register_converter(keras.layers.UpSampling1D, convert_keras_upsample_1d)
register_converter(keras.layers.UpSampling2D, convert_keras_upsample_2d)
register_converter(keras.layers.UpSampling3D, convert_keras_upsample_3d)
container.add_node('Concat', outputs, concat_output,
name=scope.get_unique_operator_name('Concat'),
op_version=1,
axis=2)
argmax_output = scope.get_unique_variable_name('argmax_output_tensor')
container.add_node('ArgMax', concat_output, argmax_output,
name=scope.get_unique_operator_name('ArgMax'),
op_version=1,
axis=2,
keepdims=0)
container.add_node('Cast', argmax_output, operator.output_full_names,
name=scope.get_unique_operator_name('Cast'),
op_version=6,
to=1)
register_converter('pyspark.ml.feature.Bucketizer', convert_bucketizer)
def calculate_bucketizer_output_shapes(operator):
check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
check_input_and_output_types(operator,
good_input_types=[FloatTensorType],
good_output_types=[FloatTensorType])
input_type = copy.deepcopy(operator.inputs[0].type)
for output in operator.outputs:
output.type = input_type
register_shape_calculator('pyspark.ml.feature.Bucketizer', calculate_bucketizer_output_shapes)