Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_pipeline_simple_mutate_inverse_transform():
expected_tape = ["1", "2", "3", "4", "4", "3", "2", "1"]
tape = TapeCallbackFunction()
p = Pipeline([
Identity(),
TransformCallbackStep(tape.callback, ["1"]),
TransformCallbackStep(tape.callback, ["2"]),
TransformCallbackStep(tape.callback, ["3"]),
TransformCallbackStep(tape.callback, ["4"]),
Identity()
])
p, _ = p.fit_transform(np.ones((1, 1)))
print("[mutating]")
p = p.mutate(new_method="inverse_transform", method_to_assign_to="transform")
p.transform(np.ones((1, 1)))
assert expected_tape == tape.get_name_tape()
def test_pipeline_nested_mutate_inverse_transform():
expected_tape = ["1", "2", "3", "4", "5", "6", "7", "7", "6", "5", "4", "3", "2", "1"]
tape = TapeCallbackFunction()
p = Pipeline([
Identity(),
TransformCallbackStep(tape.callback, ["1"]),
TransformCallbackStep(tape.callback, ["2"]),
Pipeline([
Identity(),
TransformCallbackStep(tape.callback, ["3"]),
TransformCallbackStep(tape.callback, ["4"]),
TransformCallbackStep(tape.callback, ["5"]),
Identity()
]),
TransformCallbackStep(tape.callback, ["6"]),
TransformCallbackStep(tape.callback, ["7"]),
Identity()
])
p, _ = p.fit_transform(np.ones((1, 1))) # will add range(1, 8) to tape.
print("[mutating]")
p = p.mutate(new_method="inverse_transform", method_to_assign_to="transform")
p.transform(np.ones((1, 1))) # will add reversed(range(1, 8)) to tape.
print(expected_tape)
print(tape.get_name_tape())
def create_test_case_multiple_steps_choosen():
a_callback = TapeCallbackFunction()
b_callback = TapeCallbackFunction()
return NeuraxleTestCase(
pipeline=Pipeline([
ChooseOneOrManyStepsOf([
('a', TransformCallbackStep(a_callback, transform_function=lambda di: di * 2)),
('b', TransformCallbackStep(b_callback, transform_function=lambda di: di * 2))
]),
]),
callbacks=[a_callback, b_callback],
expected_callbacks_data=[DATA_INPUTS, DATA_INPUTS],
hyperparams={
'ChooseOneOrManyStepsOf__a__enabled': True,
'ChooseOneOrManyStepsOf__b__enabled': True
},
hyperparams_space={
'ChooseOneOrManyStepsOf__a__enabled': Boolean(),
'ChooseOneOrManyStepsOf__b__enabled': Boolean()
},
expected_processed_outputs=np.array([0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18])
)
def create_test_case_invalid_step_not_choosen():
a_callback = TapeCallbackFunction()
b_callback = TapeCallbackFunction()
return NeuraxleTestCase(
pipeline=Pipeline([
ChooseOneOrManyStepsOf([
('a', TransformCallbackStep(a_callback, transform_function=lambda di: di * 2)),
('b', TransformCallbackStep(b_callback, transform_function=lambda di: di * 2))
]),
]),
callbacks=[a_callback, b_callback],
expected_callbacks_data=[DATA_INPUTS, DATA_INPUTS],
hyperparams={
'ChooseOneOrManyStepsOf__c__enabled': False,
'ChooseOneOrManyStepsOf__b__enabled': False
},
hyperparams_space={
'ChooseOneOrManyStepsOf__a__enabled': Boolean(),
'ChooseOneOrManyStepsOf__b__enabled': Boolean()
},
expected_processed_outputs=np.array(range(10))
)
def create_pipeline(tmpdir, pickle_checkpoint_step, tape, hyperparameters=None, different=False, save_pipeline=True):
if different:
pipeline = ResumablePipeline(
steps=[
('a',
DifferentCallbackStep(tape.callback, ["1"], hyperparams=hyperparameters)),
('pickle_checkpoint', pickle_checkpoint_step),
('c', TransformCallbackStep(tape.callback, ["2"])),
('d', TransformCallbackStep(tape.callback, ["3"]))
],
cache_folder=tmpdir
)
else:
pipeline = ResumablePipeline(
steps=[
('a',
TransformCallbackStep(tape.callback, ["1"], hyperparams=hyperparameters)),
('pickle_checkpoint', pickle_checkpoint_step),
('c', TransformCallbackStep(tape.callback, ["2"])),
('d', TransformCallbackStep(tape.callback, ["3"]))
], cache_folder=tmpdir
)
return pipeline
if different:
pipeline = ResumablePipeline(
steps=[
('a',
DifferentCallbackStep(tape.callback, ["1"], hyperparams=hyperparameters)),
('pickle_checkpoint', pickle_checkpoint_step),
('c', TransformCallbackStep(tape.callback, ["2"])),
('d', TransformCallbackStep(tape.callback, ["3"]))
],
cache_folder=tmpdir
)
else:
pipeline = ResumablePipeline(
steps=[
('a',
TransformCallbackStep(tape.callback, ["1"], hyperparams=hyperparameters)),
('pickle_checkpoint', pickle_checkpoint_step),
('c', TransformCallbackStep(tape.callback, ["2"])),
('d', TransformCallbackStep(tape.callback, ["3"]))
], cache_folder=tmpdir
)
return pipeline
def create_test_case_single_step_choosen():
a_callback = TapeCallbackFunction()
b_callback = TapeCallbackFunction()
return NeuraxleTestCase(
pipeline=Pipeline([
ChooseOneOrManyStepsOf([
('a', TransformCallbackStep(a_callback, transform_function=lambda di: di * 2)),
('b', TransformCallbackStep(b_callback, transform_function=lambda di: di * 2))
]),
]),
callbacks=[a_callback, b_callback],
expected_callbacks_data=[
DATA_INPUTS,
[]
],
hyperparams={
'ChooseOneOrManyStepsOf__a__enabled': True,
'ChooseOneOrManyStepsOf__b__enabled': False
},
hyperparams_space={
'ChooseOneOrManyStepsOf__a__enabled': Boolean(),
'ChooseOneOrManyStepsOf__b__enabled': Boolean()
},
import numpy as np
from neuraxle.pipeline import MiniBatchSequentialPipeline, Joiner
from neuraxle.steps.misc import TransformCallbackStep, TapeCallbackFunction, FitTransformCallbackStep
class MultiplyBy2TransformCallbackStep(TransformCallbackStep):
def transform(self, data_inputs):
TransformCallbackStep.transform(self, data_inputs)
return list(np.array(data_inputs) * 2)
def test_mini_batch_sequential_pipeline_should_transform_steps_sequentially_for_each_barrier_for_each_batch():
# Given
tape1 = TapeCallbackFunction()
tape2 = TapeCallbackFunction()
tape3 = TapeCallbackFunction()
tape4 = TapeCallbackFunction()
p = MiniBatchSequentialPipeline([
MultiplyBy2TransformCallbackStep(tape1, ["1"]),
MultiplyBy2TransformCallbackStep(tape2, ["2"]),
Joiner(batch_size=10),
def create_test_case_multiple_steps_choosen():
a_callback = TapeCallbackFunction()
b_callback = TapeCallbackFunction()
return NeuraxleTestCase(
pipeline=Pipeline([
ChooseOneOrManyStepsOf([
('a', TransformCallbackStep(a_callback, transform_function=lambda di: di * 2)),
('b', TransformCallbackStep(b_callback, transform_function=lambda di: di * 2))
]),
]),
callbacks=[a_callback, b_callback],
expected_callbacks_data=[DATA_INPUTS, DATA_INPUTS],
hyperparams={
'ChooseOneOrManyStepsOf__a__enabled': True,
'ChooseOneOrManyStepsOf__b__enabled': True
},
hyperparams_space={
'ChooseOneOrManyStepsOf__a__enabled': Boolean(),
'ChooseOneOrManyStepsOf__b__enabled': Boolean()
},
expected_processed_outputs=np.array([0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18])
)