Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
assign.value = self._array_sizes[arr][0]
# len(A) if A is 1D_Var
if fdef == ('len', 'builtins') and rhs.args and self._is_1D_Var_arr(rhs.args[0].name):
arr_var = rhs.args[0]
out = self._gen_1D_Var_len(arr_var)
out[-1].target = assign.target
self.oneDVar_len_vars[assign.target.name] = arr_var
if (sdc.config._has_h5py and (func_mod == 'sdc.io.pio_api'
and func_name in ('h5read', 'h5write', 'h5read_filter'))
and self._is_1D_arr(rhs.args[5].name)):
# TODO: make create_dataset/create_group collective
arr = rhs.args[5].name
ndims = len(self._array_starts[arr])
starts_var = ir.Var(scope, mk_unique_var("$h5_starts"), loc)
self.state.typemap[starts_var.name] = types.UniTuple(
types.int64, ndims)
start_tuple_call = ir.Expr.build_tuple(
self._array_starts[arr], loc)
starts_assign = ir.Assign(start_tuple_call, starts_var, loc)
rhs.args[2] = starts_var
counts_var = ir.Var(scope, mk_unique_var("$h5_counts"), loc)
self.state.typemap[counts_var.name] = types.UniTuple(
types.int64, ndims)
count_tuple_call = ir.Expr.build_tuple(
self._array_counts[arr], loc)
counts_assign = ir.Assign(count_tuple_call, counts_var, loc)
out = [starts_assign, counts_assign, assign]
rhs.args[3] = counts_var
rhs.args[4] = self._set1_var
# set parallel arg in file open
gufunc_ir = compiler.run_frontend(gufunc_func)
if config.DEBUG_ARRAY_OPT:
print("gufunc_ir dump ", type(gufunc_ir))
gufunc_ir.dump()
print("loop_body dump ", type(loop_body))
_print_body(loop_body)
# rename all variables in gufunc_ir afresh
var_table = get_name_var_table(gufunc_ir.blocks)
new_var_dict = {}
reserved_names = [sentinel_name] + \
list(param_dict.values()) + legal_loop_indices
for name, var in var_table.items():
if not (name in reserved_names):
new_var_dict[name] = mk_unique_var(name)
replace_var_names(gufunc_ir.blocks, new_var_dict)
if config.DEBUG_ARRAY_OPT:
print("gufunc_ir dump after renaming ")
gufunc_ir.dump()
gufunc_param_types = [
numba.types.npytypes.Array(
index_var_typ, 1, "C")] + param_types
if config.DEBUG_ARRAY_OPT:
print(
"gufunc_param_types = ",
type(gufunc_param_types),
"\n",
gufunc_param_types)
gufunc_stub_last_label = max(gufunc_ir.blocks.keys()) + 1
def _gen_reduce(self, reduce_var, reduce_op, scope, loc):
op_var = ir.Var(scope, mk_unique_var("$reduce_op"), loc)
self.state.typemap[op_var.name] = types.int32
op_assign = ir.Assign(ir.Const(reduce_op.value, loc), op_var, loc)
def f(val, op): # pragma: no cover
sdc.distributed_api.dist_reduce(val, op)
f_ir = compile_to_numba_ir(f,
{'sdc': sdc},
self.state.typingctx,
(self.state.typemap[reduce_var.name], types.int32),
self.state.typemap,
self.state.calltypes)
_, block = f_ir.blocks.popitem()
replace_arg_nodes(block, [reduce_var, op_var])
dist_reduce_nodes = [op_assign] + block.body[:-3]
def gen_wrap_if_not_known(val, val_typ, known):
if not known:
var = ir.Var(scope, mk_unique_var("var"), loc)
var_typ = types.intp
new_value = ir.Expr.call(wrap_var, [val, dsize], {}, loc)
self._define(equiv_set, var, var_typ, new_value)
self.calltypes[new_value] = sig
return (var, var_typ, new_value)
else:
return (val, val_typ, None)
var1, var1_typ, value1 = gen_wrap_if_not_known(lhs, lhs_typ, lhs_known)
var2, var2_typ, value2 = gen_wrap_if_not_known(rhs, rhs_typ, rhs_known)
post_wrap_size_var = ir.Var(scope, mk_unique_var("post_wrap_slice_size"), loc)
post_wrap_size_val = ir.Expr.binop(operator.sub, var2, var1, loc=loc)
self.calltypes[post_wrap_size_val] = signature(slice_typ, var2_typ, var1_typ)
self._define(equiv_set, post_wrap_size_var, slice_typ, post_wrap_size_val)
stmts.append(ir.Assign(value=size_val, target=size_var, loc=loc))
stmts.append(ir.Assign(value=wrap_def, target=wrap_var, loc=loc))
if value1 is not None:
stmts.append(ir.Assign(value=value1, target=var1, loc=loc))
if value2 is not None:
stmts.append(ir.Assign(value=value2, target=var2, loc=loc))
stmts.append(ir.Assign(value=post_wrap_size_val, target=post_wrap_size_var, loc=loc))
# rel_map keeps a map of relative sizes that we have seen so
# that if we compute the same relative sizes different times
# in different ways we can associate those two instances
# of the same relative size to the same equivalence class.
def _run_unary(self, assign, rhs):
arg = rhs.value
typ = self.state.typemap[arg.name]
if isinstance(typ, SeriesType):
nodes = []
arg = self._get_series_data(arg, nodes)
rhs.value = arg
self._convert_series_calltype(rhs)
out_data = ir.Var(
arg.scope, mk_unique_var(assign.target.name + '_data'), rhs.loc)
self.state.typemap[out_data.name] = self.state.calltypes[rhs].return_type
nodes.append(ir.Assign(rhs, out_data, rhs.loc))
return self._replace_func(
lambda data: sdc.hiframes.api.init_series(data),
[out_data],
pre_nodes=nodes
)
return [assign]
if op == '/':
func_typ, ir_expr = _gen_np_divide(
arg_vars[0], arg_vars[1], out_ir, typemap)
else:
func_typ = find_op_typ(op, [el_typ1])
ir_expr = ir.Expr.unary(op, arg_vars[0], loc)
calltypes[ir_expr] = func_typ
el_typ = func_typ.return_type
out_ir.append(ir.Assign(ir_expr, expr_out_var, loc))
for T in array_analysis.MAP_TYPES:
if isinstance(op, T):
# elif isinstance(op, (np.ufunc, DUFunc)):
# function calls are stored in variables which are not removed
# op is typing_key to the variables type
func_var_name = _find_func_var(typemap, op, avail_vars)
func_var = ir.Var(scope, mk_unique_var(func_var_name), loc)
typemap[func_var.name] = typemap[func_var_name]
func_var_def = func_ir.get_definition(func_var_name)
if isinstance(func_var_def, ir.Expr) and func_var_def.op == 'getattr' and func_var_def.attr == 'sqrt':
g_math_var = ir.Var(scope, mk_unique_var("$math_g_var"), loc)
typemap[g_math_var.name] = types.misc.Module(math)
g_math = ir.Global('math', math, loc)
g_math_assign = ir.Assign(g_math, g_math_var, loc)
func_var_def = ir.Expr.getattr(g_math_var, 'sqrt', loc)
out_ir.append(g_math_assign)
# out_ir.append(func_var_def)
ir_expr = ir.Expr.call(func_var, arg_vars, (), loc)
call_typ = typemap[func_var.name].get_call_type(
typing.Context(), [el_typ] * len(arg_vars), {})
calltypes[ir_expr] = call_typ
el_typ = call_typ.return_type
#signature(el_typ, el_typ)
self.calltypes[getitem_call] = signature(subarr_typ, arr_typ,
index_typ)
init_block.append(ir.Assign(getitem_call, subarr_var, loc))
target = subarr_var
else:
# Otherwise it is a boolean array that is used as index.
assert(isinstance(index_typ, types.ArrayCompatible))
size_vars = equiv_set.get_shape(target)
bool_typ = index_typ.dtype
# generate loopnests and size variables from lhs correlations
loopnests = []
index_vars = []
for size_var in size_vars:
index_var = ir.Var(scope, mk_unique_var("parfor_index"), loc)
index_vars.append(index_var)
self.typemap[index_var.name] = types.uintp
loopnests.append(LoopNest(index_var, 0, size_var, 1))
# generate body
body_label = next_label()
body_block = ir.Block(scope, loc)
index_var, index_var_typ = self._make_index_var(
scope, index_vars, body_block)
parfor = Parfor(loopnests, init_block, {}, loc, index_var, equiv_set,
('setitem',), self.flags)
if shape:
# slice subarray
parfor.loop_body = {body_label: body_block}
true_block = body_block
end_label = None
'hpat': hpat}, self.typingctx,
(self.typemap[data.name], types.intp),
self.typemap, self.calltypes).blocks.popitem()[1]
replace_arg_nodes(f_block, [data, ind_var])
nodes += f_block.body[:-2]
nodes[-1].target = assign.target
return nodes
if isinstance(self.typemap[rhs.value.name], SeriesType):
rhs.value = self._get_series_data(rhs.value, nodes)
self._convert_series_calltype(rhs)
lhs = assign.target
# convert output to Series from Array
if isinstance(self.typemap[lhs.name], SeriesType):
new_lhs = ir.Var(
lhs.scope, mk_unique_var(lhs.name + '_data'), lhs.loc)
self.typemap[new_lhs.name] = series_to_array_type(
self.typemap[lhs.name])
nodes.append(ir.Assign(rhs, new_lhs, lhs.loc))
return self._replace_func(
lambda A: hpat.hiframes.api.init_series(A), [new_lhs],
pre_nodes=nodes)
nodes.append(assign)
return nodes
nodes.append(assign)
return nodes
df_var = self._get_df_obj_select(grp_var, 'groupby')
df_type = self.state.typemap[df_var.name]
out_typ = self.state.typemap[lhs.name]
nodes = []
in_vars = {c: self._get_dataframe_data(df_var, c, nodes)
for c in grp_typ.selection}
in_key_arrs = [self._get_dataframe_data(df_var, c, nodes)
for c in grp_typ.keys]
out_key_vars = None
if grp_typ.as_index is False:
out_key_vars = []
for k in grp_typ.keys:
out_key_var = ir.Var(lhs.scope, mk_unique_var(k), lhs.loc)
ind = out_typ.columns.index(k)
self.state.typemap[out_key_var.name] = out_typ.data[ind]
out_key_vars.append(out_key_var)
df_col_map = {}
for c in grp_typ.selection:
var = ir.Var(lhs.scope, mk_unique_var(c), lhs.loc)
self.state.typemap[var.name] = (out_typ.data
if isinstance(out_typ, SeriesType)
else out_typ.data[out_typ.columns.index(c)])
df_col_map[c] = var
agg_func = get_agg_func(self.state.func_ir, func_name, rhs)
agg_node = hiframes.aggregate.Aggregate(
lhs.name, df_var.name, grp_typ.keys, out_key_vars, df_col_map,
getitem_assign = ir.Assign(getitem_call, X_val, loc)
# v_val = getitem(V, inner_index)
v_val = ir.Var(scope, mk_unique_var("$" + in2.name + "_val"), loc)
typemap[v_val.name] = el_typ
v_getitem_call = ir.Expr.getitem(in2, inner_index, loc)
calltypes[v_getitem_call] = signature(
el_typ, typemap[in2.name], types.intp)
v_getitem_assign = ir.Assign(v_getitem_call, v_val, loc)
# add_var = X_val * v_val
add_var = ir.Var(scope, mk_unique_var("$add_var"), loc)
typemap[add_var.name] = el_typ
add_call = ir.Expr.binop('*', X_val, v_val, loc)
calltypes[add_call] = signature(el_typ, el_typ, el_typ)
add_assign = ir.Assign(add_call, add_var, loc)
# acc_var = sum_var + add_var
acc_var = ir.Var(scope, mk_unique_var("$acc_var"), loc)
typemap[acc_var.name] = el_typ
acc_call = ir.Expr.inplace_binop('+=', '+', sum_var, add_var, loc)
calltypes[acc_call] = signature(el_typ, el_typ, el_typ)
acc_assign = ir.Assign(acc_call, acc_var, loc)
# sum_var = acc_var
final_assign = ir.Assign(acc_var, sum_var, loc)
# jump to header
b_jump_header = ir.Jump(-1, loc)
body_block.body = [
inner_index_assign,
tuple_assign,
getitem_assign,
v_getitem_assign,
add_assign,
acc_assign,
final_assign,