Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __getattr__(self, key):
if key == 'iteritems' and hasattr(super(ParameterMapper, self), 'items'):
return super(ParameterMapper, self).items
if hasattr(super(ParameterMapper, self), key):
return getattr(super(ParameterMapper, self), key)
if key not in self:
raise _user_exceptions.FlyteAssertion("{} doesn't exist.".format(key))
return self[key]
def __setattr__(self, key, value):
if '_initialized' in self.__dict__:
raise _user_exceptions.FlyteAssertion("Parameters are immutable.")
else:
super(ParameterMapper, self).__setattr__(key, value)
def outputs(self):
"""
Returns the outputs of the task execution, if available, in the standard Python format that is produced by
the type engine. If not available, perhaps due to execution being in progress or an error being produced,
this will raise an exception.
:rtype: dict[Text, T]
"""
if not self.is_complete:
raise _user_exceptions.FlyteAssertion("Please what until the task execution has completed before "
"requesting the outputs.")
if self.error:
raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.")
if self._outputs is None:
self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(
_engine_loader.get_engine().get_task_execution(self).get_outputs()
)
return self._outputs
def to_flyte_idl(self):
if self.input_mode == _sdk_sagemaker_types.InputMode.FILE:
input_mode = _training_job.InputMode.FILE
elif self.input_mode == _sdk_sagemaker_types.InputMode.PIPE:
input_mode = _training_job.InputMode.PIPE
else:
raise _user_exceptions.FlyteValidationException(
"Invalid SageMaker Input Mode Specified: [{}]".format(self.input_mode))
if self.algorithm_name == _sdk_sagemaker_types.AlgorithmName.CUSTOM:
alg_name = _training_job.AlgorithmName.CUSTOM
elif self.algorithm_name == _sdk_sagemaker_types.AlgorithmName.XGBOOST:
alg_name = _training_job.AlgorithmName.XGBOOST
else:
raise _user_exceptions.FlyteValidationException(
"Invalid SageMaker Algorithm Name Specified: [{}]".format(self.algorithm_name))
return _training_job.AlgorithmSpecification(
input_mode=input_mode,
algorithm_name=alg_name,
algorithm_version=self.algorithm_version,
metric_definitions=[m.to_flyte_idl() for m in self.metric_definitions],
)
:rtype: None
"""
poll_interval = poll_interval or _datetime.timedelta(seconds=30)
if timeout is None:
time_to_give_up = _datetime.datetime.max
else:
time_to_give_up = _datetime.datetime.utcnow() + timeout
self._sync_closure()
while _datetime.datetime.utcnow() < time_to_give_up:
if self.is_complete:
self.sync()
return
_time.sleep(poll_interval.total_seconds())
self._sync_closure()
raise _user_exceptions.FlyteTimeout("Execution {} did not complete before timeout.".format(self))
memory_limit,
discoverable,
timeout,
environment,
pod_spec=None,
primary_container_name=None):
"""
:param _sdk_runnable.SdkRunnableTask sdk_runnable_task:
:param generated_pb2.PodSpec pod_spec:
:param Text primary_container_name:
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
if not pod_spec:
raise _user_exceptions.FlyteValidationException("A pod spec cannot be undefined")
if not primary_container_name:
raise _user_exceptions.FlyteValidationException("A primary container name cannot be undefined")
super(SdkSidecarTask, self).__init__(
task_function,
task_type,
discovery_version,
retries,
deprecated,
storage_request,
cpu_request,
gpu_request,
memory_request,
storage_limit,
cpu_limit,
gpu_limit,
memory_limit,
discoverable,
should take action themselves or pass on to the platform owners. We will dispatch metrics and such appropriately.
"""
try:
_CONTEXT_STACK.append(_SYSTEM_CONTEXT)
if _is_base_context():
try:
return wrapped(*args, **kwargs)
except FlyteScopedException as ex:
_reraise(ex.type, ex.value, ex.traceback)
else:
try:
return wrapped(*args, **kwargs)
except FlyteScopedException:
# Just pass-on the exception that is already wrapped and scoped
_reraise(*_exc_info())
except _user_exceptions.FlyteUserException:
# Re-raise from here.
_reraise(
FlyteScopedUserException,
FlyteScopedUserException(*_exc_info()),
_exc_info()[2])
except:
# System error, raise full stack-trace all the way up the chain.
_reraise(
FlyteScopedSystemException,
FlyteScopedSystemException(*_exc_info(), kind=_error_model.ContainerError.Kind.RECOVERABLE),
_exc_info()[2])
finally:
_CONTEXT_STACK.pop()
def error(self):
"""
If execution is in progress, raise an exception. Otherwise, return None if no error was present upon
reaching completion.
:rtype: flytekit.models.core.execution.ExecutionError or None
"""
if not self.is_complete:
raise _user_exceptions.FlyteAssertion("Please wait until a workflow has completed before checking for an "
"error.")
return self.closure.error
def _extract_pair(identifier_file, object_file):
"""
:param Text identifier_file:
:param Text object_file:
:rtype: (flyteidl.core.identifier_pb2.Identifier, T)
"""
resource_map = {
_identifier_pb2.LAUNCH_PLAN: _launch_plan_pb2.LaunchPlanSpec,
_identifier_pb2.WORKFLOW: _workflow_pb2.WorkflowSpec,
_identifier_pb2.TASK: _task_pb2.TaskSpec
}
id = _load_proto_from_file(_identifier_pb2.Identifier, identifier_file)
if not id.resource_type in resource_map:
raise _user_exceptions.FlyteAssertion(f"Resource type found in identifier {id.resource_type} invalid, must be launch plan, "
f"task, or workflow")
entity = _load_proto_from_file(resource_map[id.resource_type], object_file)
return id, entity
if sdk_task.type in {
_sdk_constants.SdkTaskType.PYTHON_TASK,
_sdk_constants.SdkTaskType.SPARK_TASK,
_sdk_constants.SdkTaskType.SENSOR_TASK,
}:
return ReturnOutputsTask(sdk_task)
elif sdk_task.type in {
_sdk_constants.SdkTaskType.DYNAMIC_TASK,
}:
return DynamicTask(sdk_task)
elif sdk_task.type in {
_sdk_constants.SdkTaskType.BATCH_HIVE_TASK,
}:
return HiveTask(sdk_task)
else:
raise _user_exceptions.FlyteAssertion(
"Unit tests are not currently supported for tasks of type: {}".format(
sdk_task.type
)