Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_init_raises_if_file_path_doesnt_exist():
path = 'this-file-doesnt-exist.json'
with pytest.raises(exceptions.DataPackageException):
Package(path)
def test_validate_should_raise_when_invalid():
schema_dict = {
'properties': {
'name': {
'type': 'string',
}
},
'required': ['name'],
}
data = {}
schema = Profile(schema_dict)
with pytest.raises(exceptions.ValidationError):
schema.validate(data)
def test_zip_with_resources_outside_base_path_isnt_safe(self, tmpfile):
descriptor = {
'resources': [
{'path': __file__},
]
}
with zipfile.ZipFile(tmpfile.name, 'w') as z:
z.writestr('datapackage.json', json.dumps(descriptor))
with pytest.raises(datapackage.exceptions.DataPackageException):
dp = datapackage.DataPackage(tmpfile.name, {})
DataPackage -- valid DataPackage instance
"""
if not exists('datapackage.json'):
print('Current directory is not a datapackage: datapackage.json not found.')
sys.exit(1)
try:
dp = datapackage.DataPackage('datapackage.json')
except:
print('datapackage.json is malformed')
sys.exit(1)
try:
dp.validate()
except datapackage.exceptions.ValidationError:
for error in dp.iter_errors():
# TODO: printing error looks very noisy on output, maybe try make it look nice.
print(error)
sys.exit(1)
return dp
def _load_and_validate_datapackage(url=None, upload=None):
try:
if _upload_attribute_is_valid(upload):
dp = datapackage.DataPackage(upload.file)
else:
dp = datapackage.DataPackage(url)
dp.validate()
except (datapackage.exceptions.DataPackageException,
datapackage.exceptions.SchemaError,
datapackage.exceptions.ValidationError) as e:
msg = {'datapackage': [e.message]}
raise toolkit.ValidationError(msg)
if not dp.safe():
msg = {'datapackage': ['the Data Package has unsafe attributes']}
raise toolkit.ValidationError(msg)
return dp
# Raises
DataPackageException: base class of any error
CastError: data cast error
IntegrityError: integrity checking error
UniqueKeyError: unique key constraint violation
UnresolvedFKError: unresolved foreign key reference error
# Returns
Iterator[list]: yields rows
"""
# Error for non tabular
if not self.tabular:
message = 'Methods iter/read are not supported for non tabular data'
raise exceptions.DataPackageException(message)
# Get integrity
if integrity:
integrity = self.__get_integrity()
# Get relations
if relations:
relations = self.__get_relations()
return self.__get_table().iter(
integrity=integrity, relations=relations, **options)
FOREIGN_KEYS = [
{"fields": cmso.OBJECT_ID,
"reference": {"fields": cmso.OBJECT_ID,
"resource": cmso.OBJECTS_TABLE}}
]
def is_tabular(dp):
if not isinstance(dp, datapackage.DataPackage):
return False
reg = datapackage.registry.Registry()
return dp.schema.to_dict() == reg.get('tabular')
class ValidationError(datapackage.exceptions.ValidationError):
pass
class Validator(object):
def __init__(self, log_level=None):
self.logger = get_logger(self.__class__.__name__, level=log_level)
def validate(self, dp):
if isinstance(dp, datapackage.DataPackage) and not is_tabular(dp):
raise ValueError("data package must be a tabular data package")
else:
dp = datapackage.DataPackage(dp, schema="tabular")
dp.validate()
self.logger.debug("valid tabular data package")
if len(dp.resources) < 2:
for bus, kwargs in sorted(element["inputs"].items())},
"outputs": {
data["buses"][bus]: flow(**remap(kwargs, attributemap, flow))
for bus, kwargs in sorted(element["outputs"].items())}},
resolve_object_references(element["parameters"],
f=lambda r: r == "buses"))
for name, element in sorted(data["elements"].items())
for flow in (typemap.get(FLOW_TYPE, HSN),)}
facades = {}
for r in package.resources:
if all(re.match(r"^data/elements/.*$", p)
for p in listify(r.descriptor["path"], 1)):
try:
facade_data = r.read(keyed=True, relations=True)
except dp.exceptions.CastError:
raise dp.exceptions.LoadError((cast_error_msg).format(r.name))
except Exception as e:
raise dp.exceptions.LoadError(
("Could not read data for resource with name `{}`. "
" Maybe wrong foreign keys?\n"
"Exception was: {}").format(r.name, e))
foreign_keys = {
fk["fields"]: fk["reference"]
for fk in r.descriptor["schema"].get("foreignKeys", ())}
for facade in facade_data:
# convert decimal to float
for f, v in facade.items():
if isinstance(v, Decimal):
facade[f] = float(v)
read_facade(facade, facades, create, typemap, data, objects,
# Inspect source
self.__source_inspection = _inspect_source(
self.__current_descriptor.get('data'),
self.__current_descriptor.get('path'),
self.__base_path,
self.__storage)
# Instantiate profile
self.__profile = Profile(self.__current_descriptor.get('profile'))
# Validate descriptor
try:
self.__profile.validate(self.__current_descriptor)
self.__errors = []
except exceptions.ValidationError as exception:
self.__errors = exception.errors
if self.__strict:
raise exception