Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_save_config(hook=None):
if hook is None:
save_config_collection = SaveConfig(save_steps=[4, 5, 6])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/" + run_id
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
hook = t_hook(
out_dir=out_dir,
save_config=save_config,
include_collections=["ReluActivation", "weights", "biases", "gradients", "default"],
)
custom_collect = hook.get_collection("ReluActivation")
custom_collect.save_config = save_config_collection
custom_collect.include(["relu*", "input_*", "output*"])
run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)
if hook is None:
shutil.rmtree(out_dir)
print("Registering the hook with out_dir {0}".format(out_dir))
hook = t_hook(
out_dir=out_dir,
save_config=global_save_config,
include_collections=[
"weights",
"biases",
"gradients",
"default",
"ReluActivation",
"flatten",
],
reduction_config=global_reduce_config,
)
hook.get_collection("ReluActivation").include(["relu*"])
hook.get_collection("ReluActivation").save_config = SaveConfig(save_steps=[4, 5, 6])
hook.get_collection("ReluActivation").reduction_config = ReductionConfig(
reductions=["min"], abs_reductions=["max"]
)
hook.get_collection("flatten").include(["flatten*"])
hook.get_collection("flatten").save_config = SaveConfig(save_steps=[4, 5, 6])
hook.get_collection("flatten").reduction_config = ReductionConfig(
norms=["l1"], abs_norms=["l2"]
)
run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)
# Testing
print("Created the trial with out_dir {0}".format(out_dir))
tr = create_trial(out_dir)
assert tr
def test_save_all(hook=None, out_dir=None):
hook_created = False
if hook is None:
hook_created = True
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/" + run_id
print("Registering the hook with out_dir {}".format(out_dir))
hook = t_hook(out_dir=out_dir, save_config=save_config, save_all=True)
run_mnist_gluon_model(hook=hook, num_steps_train=7, num_steps_eval=5)
# assert for steps and tensor_names
print("Created the trial with out_dir {}".format(out_dir))
tr = create_trial(out_dir)
tensor_list = tr.tensor_names()
assert tr
assert len(tr.steps()) == 4
# some tensor names, like input and output, can't be retrieved from training session, so here we only assert for tensor numbers
# 46 is gotten from index file
# if no assertion failure, then the script could save all tensors
assert len(tensor_list) == 46
if hook_created:
def create_hook(output_s3_uri):
# With the following SaveConfig, we will save tensors for steps 1, 2 and 3
# (indexing starts with 0).
save_config = SaveConfig(save_steps=[1, 2, 3])
# Create a hook that logs weights, biases and gradients while training the model.
hook = Hook(
out_dir=output_s3_uri,
save_config=save_config,
include_collections=["weights", "gradients", "biases"],
)
return hook
def create_hook(output_s3_uri):
save_config = SaveConfig(save_interval=1)
custom_collect = smd.get_collection("ReluActivation")
custom_collect.save_config = save_config
custom_collect.include([".*relu_output"])
hook = Hook(
out_dir=output_s3_uri, save_config=save_config, include_collections=["ReluActivation"]
)
return hook
def test_save_config(hook=None, out_dir=None):
hook_created = False
if hook is None:
hook_created = True
global_reduce_config = ReductionConfig(reductions=["max", "mean"])
global_save_config = SaveConfig(save_steps=[0, 1, 2, 3])
run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
out_dir = "/tmp/newlogsRunTest/" + run_id
print("Registering the hook with out_dir {0}".format(out_dir))
hook = t_hook(
out_dir=out_dir,
save_config=global_save_config,
include_collections=[
"weights",
"biases",
"gradients",
"default",
"ReluActivation",
"flatten",
],
reduction_config=global_reduce_config,
def create_hook(output_s3_uri):
save_config = SaveConfig(save_interval=1)
custom_collect = smd.get_collection("inputData")
custom_collect.save_config = save_config
custom_collect.include([".*hybridsequential0_input_0"])
hook = Hook(out_dir=output_s3_uri, save_config=save_config, include_collections=["inputData"])
return hook
def create_hook(output_s3_uri):
# With the following SaveConfig, we will save tensors for steps 0, 1, 2 and 3
# (indexing starts with 0).
save_config = SaveConfig(save_steps=[0, 1, 2, 3])
# Create a hook that logs weights, biases and gradients while training the model.
hook = Hook(
out_dir=output_s3_uri,
save_config=save_config,
include_collections=["ReluActivation", "weights", "biases", "gradients"],
)
hook.get_collection("ReluActivation").include(["relu*", "input_*"])
return hook