Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_nxgraph_full(self):
sio = io.StringIO()
edge_types = (
registry.entity.name,
registry.email.name,
registry.phone.name,
)
exporter = NXGraphExporter(sio, edge_types=edge_types)
for entity in ENTITIES:
proxy = model.get_proxy(entity)
exporter.write(proxy)
self.assertEqual(len(exporter.graph.nodes), 5)
self.assertEqual(len(exporter.graph.edges), 4)
def map(self, record, entities):
proxy = self.model.make_entity(self.schema)
proxy.id = self.compute_key(record)
if proxy.id is None:
return
# THIS IS HACKY
# Some of the converters, e.g. for phone numbers, work better if they
# know the country which the number is from. In order to provide that
# detail, we are first running country fields, then making the data
# from that accessible to phone and address parsers.
for prop in self.properties:
if prop.prop.type == registry.country:
prop.map(proxy, record, entities)
for prop in self.properties:
if prop.prop.type != registry.country:
prop.map(proxy, record, entities)
for prop in self.properties:
if prop.required and not proxy.has(prop.prop):
# This is a bit weird, it flags fields to be required in
# the mapping, not in the model. Basically it means: if
# this row of source data doesn't have that field, then do
# not map it again.
return
return proxy
def _format_date(proxy):
dates = proxy.get_type_values(registry.date)
if not len(dates):
return ''
return min(dates)
def apply(self, proxy):
"""Rewrite an entity proxy so all IDs mentioned are limited to
the namespace.
An exception is made for sameAs declarations."""
signed = proxy.clone()
signed.id = self.sign(proxy.id)
for prop in proxy.iterprops():
if prop.type != registry.entity:
continue
for value in signed.pop(prop):
value = get_entity_id(value)
signed.add(prop, self.sign(value))
# linked.add('sameAs', proxy.id, quiet=True)
signed.remove("sameAs", signed.id)
return signed
import logging
import fingerprints
from pprint import pprint # noqa
from banal import ensure_list
from followthemoney.types import registry
from aleph.index.util import bool_query, none_query
log = logging.getLogger(__name__)
MAX_CLAUSES = 500
REQUIRED = [registry.name, registry.iban, registry.identifier]
def _make_queries(prop, value, specificity):
if prop.type == registry.name:
boost = (1 + specificity) * 2
yield {
'match': {
'names.text': {
'query': value,
'operator': 'and',
'minimum_should_match': '60%',
'boost': boost
}
}
}
fp = fingerprints.generate(value)
def to_dict(self):
return {
"schemata": {s.name: s.to_dict() for s in self.schemata.values()},
"types": {t.name: t.to_dict() for t in registry.types},
}
def _validate(self, value):
if not registry.date.validate(value):
raise ValidationError('Invalid date: %s' % value)
def __init__(self, manager, name, email):
self.email = ascii_text(stringify(email))
self.name = stringify(name)
if not registry.email.validate(self.email):
self.email = None
if registry.email.validate(self.name):
self.email = self.email or ascii_text(self.name)
self.name = None
# This should be using formataddr, but I cannot figure out how
# to use that without encoding the name.
self.label = None
if self.name is not None and self.email is not None:
self.label = '%s <%s>' % (self.name, self.email)
elif self.name is None and self.email is not None:
self.label = self.email
elif self.email is None and self.name is not None:
self.label = self.name
self.entity = None