Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_tf(sagemaker_session, tf_version):
tf = TensorFlow(
entry_point=SCRIPT_FILE,
role=ROLE,
sagemaker_session=sagemaker_session,
training_steps=1000,
evaluation_steps=10,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
framework_version=tf_version,
requirements_file=REQUIREMENTS_FILE,
source_dir=DATA_DIR,
)
inputs = "s3://mybucket/train"
tf.fit(inputs=inputs, experiment_config=EXPERIMENT_CONFIG)
def test_tf_script_mode_ps(time, strftime, sagemaker_session):
tf = TensorFlow(
entry_point=SCRIPT_FILE,
role=ROLE,
sagemaker_session=sagemaker_session,
py_version="py3",
train_instance_type=INSTANCE_TYPE,
train_instance_count=1,
framework_version="1.11",
source_dir=DATA_DIR,
distributions=DISTRIBUTION_ENABLED,
)
inputs = "s3://mybucket/train"
tf.fit(inputs=inputs)
call_names = [c[0] for c in sagemaker_session.method_calls]
assert call_names == ["train", "logs_for_job"]
def test_deploy(sagemaker_session, tf_version):
estimator = TensorFlow(
entry_point=SCRIPT,
source_dir=SOURCE_DIR,
role=ROLE,
framework_version=tf_version,
train_instance_count=2,
train_instance_type=INSTANCE_TYPE_CPU,
sagemaker_session=sagemaker_session,
base_job_name="test-cifar",
)
estimator.fit("s3://mybucket/train")
print("job succeeded: {}".format(estimator.latest_training_job.name))
estimator.deploy(initial_instance_count=1, instance_type=INSTANCE_TYPE_CPU)
image = IMAGE_URI_FORMAT_STRING.format(REGION, CPU_IMAGE_NAME, tf_version, "cpu", "py2")
sagemaker_session.create_model.assert_called_with(
def test_run_tensorboard_locally_port_in_use(
sleep, time, strftime, popen, call, access, socket, rmtree, mkdtemp, sync, sagemaker_session
):
tf = TensorFlow(
entry_point=SCRIPT_PATH,
role=ROLE,
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
)
popen().poll.side_effect = [-1, None]
tf.fit(inputs="s3://mybucket/train", run_tensorboard_locally=True)
popen.assert_any_call(
["tensorboard", "--logdir", "/my/temp/folder", "--host", "localhost", "--port", "6006"],
stderr=-1,
stdout=-1,
)
def test_mnist(sagemaker_session, ecr_image, instance_type, framework_version):
resource_path = os.path.join(os.path.dirname(__file__), '..', '..', 'resources')
script = os.path.join(resource_path, 'mnist', 'mnist.py')
estimator = TensorFlow(entry_point=script,
role='SageMakerRole',
train_instance_type=instance_type,
train_instance_count=1,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
framework_version=framework_version,
script_mode=True)
inputs = estimator.sagemaker_session.upload_data(
path=os.path.join(resource_path, 'mnist', 'data'),
key_prefix='scriptmode/mnist')
estimator.fit(inputs, job_name=unique_name_from_base('test-sagemaker-mnist'))
_assert_s3_file_exists(sagemaker_session.boto_region_name, estimator.model_data)
def test_tf_script_mode(time, strftime, sagemaker_session):
tf = TensorFlow(
entry_point=SCRIPT_FILE,
role=ROLE,
sagemaker_session=sagemaker_session,
py_version="py3",
train_instance_type=INSTANCE_TYPE,
train_instance_count=1,
framework_version="1.11",
source_dir=DATA_DIR,
)
inputs = "s3://mybucket/train"
tf.fit(inputs=inputs)
call_names = [c[0] for c in sagemaker_session.method_calls]
assert call_names == ["train", "logs_for_job"]
def test_tuning_tf_script_mode_lustre(efs_fsx_setup, sagemaker_session, cpu_instance_type):
role = efs_fsx_setup["role_name"]
subnets = [efs_fsx_setup["subnet_id"]]
security_group_ids = efs_fsx_setup["security_group_ids"]
estimator = TensorFlow(
entry_point=SCRIPT,
role=role,
train_instance_count=1,
train_instance_type=cpu_instance_type,
script_mode=True,
sagemaker_session=sagemaker_session,
py_version=PY_VERSION,
framework_version=TensorFlow.LATEST_VERSION,
subnets=subnets,
security_group_ids=security_group_ids,
)
hyperparameter_ranges = {"epochs": IntegerParameter(1, 2)}
objective_metric_name = "accuracy"
metric_definitions = [{"Name": objective_metric_name, "Regex": "accuracy = ([0-9\\.]+)"}]
tuner = HyperparameterTuner(
instance_type = "ml.c4.xlarge"
instance_count = 2
train_input = sagemaker_session.upload_data(
path=os.path.join(DATA_DIR, "iris", "data"), key_prefix="integ-test-data/tf_iris"
)
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
ec2_client = sagemaker_session.boto_session.client("ec2")
subnet_ids, security_group_id = get_or_create_vpc_resources(
ec2_client, sagemaker_session.boto_session.region_name
)
setup_security_group_for_encryption(ec2_client, security_group_id)
estimator = TensorFlow(
entry_point=script_path,
role="SageMakerRole",
framework_version=tf_full_version,
training_steps=1,
evaluation_steps=1,
hyperparameters={"input_tensor_name": "inputs"},
train_instance_count=instance_count,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
base_job_name="test-vpc-tf",
subnets=subnet_ids,
security_group_ids=[security_group_id],
encrypt_inter_container_traffic=True,
)
job_name = unique_name_from_base("test-tf-vpc-multi")
def test_tuning_tf_script_mode(sagemaker_session, cpu_instance_type, tf_full_version):
resource_path = os.path.join(DATA_DIR, "tensorflow_mnist")
script_path = os.path.join(resource_path, "mnist.py")
estimator = TensorFlow(
entry_point=script_path,
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
script_mode=True,
sagemaker_session=sagemaker_session,
py_version=PYTHON_VERSION,
framework_version=tf_full_version,
)
hyperparameter_ranges = {"epochs": IntegerParameter(1, 2)}
objective_metric_name = "accuracy"
metric_definitions = [{"Name": objective_metric_name, "Regex": "accuracy = ([0-9\\.]+)"}]
tuner = HyperparameterTuner(
estimator,
def test_framework_training_config_all_args(sagemaker_session):
tf = tensorflow.TensorFlow(
entry_point="{{ entry_point }}",
source_dir="{{ source_dir }}",
enable_cloudwatch_metrics=False,
container_log_level="{{ log_level }}",
code_location="s3://{{ bucket_name }}/{{ prefix }}",
training_steps=1000,
evaluation_steps=100,
checkpoint_path="{{ checkpoint_path }}",
py_version="py2",
framework_version="1.10.0",
requirements_file="",
role="{{ role }}",
train_instance_count="{{ instance_count }}",
train_instance_type="ml.c4.2xlarge",
train_volume_size="{{ train_volume_size }}",
train_volume_kms_key="{{ train_volume_kms_key }}",