Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _test_resource(self, url='anything', format='TXT', archived=True, cached=True, license_id='uk-ogl'):
pkg = {'license_id': license_id,
'resources': [
{'url': url, 'format': format, 'description': 'Test'}
]}
pkg = ckan_factories.Dataset(**pkg)
res_id = pkg['resources'][0]['id']
if archived:
archival = Archival.create(res_id)
archival.cache_filepath = __file__ if cached else None # just needs to exist
archival.updated = TODAY
model.Session.add(archival)
model.Session.commit()
return model.Resource.get(res_id)
If you wish to modify or delete a column, add the column name and
query to the MIGRATIONS_MODIFY which only runs if the column
does exist.
"""
from ckan import model
MIGRATIONS_ADD = OrderedDict({
"etag": "ALTER TABLE archival ADD COLUMN etag character varying",
"last_modified": "ALTER TABLE archival ADD COLUMN last_modified character varying"
})
MIGRATIONS_MODIFY = OrderedDict({
})
q = "select column_name from INFORMATION_SCHEMA.COLUMNS where table_name = 'archival';"
current_cols = list([m[0] for m in model.Session.execute(q)])
for k, v in MIGRATIONS_ADD.iteritems():
if k not in current_cols:
self.log.info(u"Adding column '{0}'".format(k))
self.log.info(u"Executing '{0}'".format(v))
model.Session.execute(v)
model.Session.commit()
for k, v in MIGRATIONS_MODIFY.iteritems():
if k in current_cols:
self.log.info(u"Removing column '{0}'".format(k))
self.log.info(u"Executing '{0}'".format(v))
model.Session.execute(v)
model.Session.commit()
self.log.info("Migrations complete")
if update:
ckan_user = toolkit.get_action(u'user_update')(
context={
u'ignore_auth': True
},
data_dict=user_dict
)
else:
ckan_user = toolkit.get_action(u'user_create')(
context={
u'ignore_auth': True
},
data_dict=user_dict
)
ldap_user = LdapUser(user_id=ckan_user[u'id'], ldap_id=ldap_user_dict[u'username'])
Session.add(ldap_user)
Session.commit()
# Add the user to it's group if needed
if u'ckanext.ldap.organization.id' in config:
toolkit.get_action(u'member_create')(
context={
u'ignore_auth': True
},
data_dict={
u'id': config[u'ckanext.ldap.organization.id'],
u'object': user_name,
u'object_type': u'user',
u'capacity': config[u'ckanext.ldap.organization.role']
}
)
return user_name
def command(self):
""" Helpful command for development """
from sqlalchemy import func
self._load_config()
self.log = logging.getLogger(__name__)
import ckan.model as model
model.Session.remove()
model.Session.configure(bind=model.meta.engine)
model.repo.new_revision()
formats = ['csv', 'xls']
if len(self.args) == 1:
formats = self.args[0].split(',')
log.info("Processing %s" % ' and '.join(formats))
for fmt in formats:
q = model.Session.query(model.Resource)\
.filter(func.lower(model.Resource.format) == func.lower(fmt))\
.filter(model.Resource.state == 'active')
total = q.count()
records = q.order_by(func.random()).limit(self.options.count).all()
self.log.info("We have %d records from %d files of %s format" %
from ckanext.archiver.model import Archival, Status
resources = common.get_resources(state='active',
publisher_ref=options.publisher,
resource_id=options.resource,
dataset_name=options.dataset)
stats = StatsList()
widgets = ['Resources: ', Percentage(), ' ', Bar(), ' ', ETA()]
progress = ProgressBar(widgets=widgets)
for res in progress(resources):
# Gather the details of archivals from TaskStatus and Resource
# to fill all properties of Archival apart from:
# * package_id
# * resource_id
fields = {}
archiver_task_status = model.Session.query(model.TaskStatus)\
.filter_by(entity_id=res.id)\
.filter_by(task_type='archiver')\
.filter_by(key='status')\
.first()
if archiver_task_status:
ats_error = json.loads(archiver_task_status.error)
fields['status_id'] = Status.by_text(archiver_task_status.value)
fields['is_broken'] = Status.is_status_broken(fields['status_id'])
fields['reason'] = ats_error['reason']
fields['last_success'] = date_str_to_datetime_or_none(ats_error['last_success'])
fields['first_failure'] = date_str_to_datetime_or_none(ats_error['first_failure'])
fields['failure_count'] = int(ats_error['failure_count'])
fields['url_redirected_to'] = ats_error['url_redirected_to']
fields['updated'] = archiver_task_status.last_updated
else:
if not (res.cache_url
def create(cls, resource_id):
c = cls()
c.resource_id = resource_id
# Find the package_id for the resource.
q = model.Session.query(model.Package.id)
if toolkit.check_ckan_version(max_version='2.2.99'):
q = q.join(model.ResourceGroup)
q = q.join(model.Resource) \
.filter_by(id=c.resource_id)
result = q.first()
if not result or not result[0]:
raise Exception("Missing dataset")
c.package_id = result[0]
return c
average_stars = round(float(total_stars) / num_pkgs_scored, 1) \
if num_pkgs_scored else 0.0
row = OrderedDict((
('organization_title', results[org_name]['organization_title']),
('organization_name', org_name),
('total_stars', total_stars),
('average_stars', average_stars),
))
row.update(jsonify_counter(org_counts['score_counts']))
table.append(row)
table.sort(key=lambda x: (-x['total_stars'],
-x['average_stars']))
# Get total number of packages & resources
num_packages = model.Session.query(model.Package)\
.filter_by(state='active')\
.count()
return {'table': table,
'total_score_counts': jsonify_counter(total_score_counts),
'num_packages_scored': sum(total_score_counts.values()),
'num_packages': num_packages,
}
def create_example_vocabs(self):
'''
Adds example vocabularies to the database if they don't already exist.
'''
user = get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
context = {'model': model, 'session': model.Session, 'user': user['name']}
try:
data = {'id': forms.GENRE_VOCAB}
get_action('vocabulary_show')(context, data)
log.info("Example genre vocabulary already exists, skipping.")
except NotFound:
log.info("Creating vocab %s" % forms.GENRE_VOCAB)
data = {'name': forms.GENRE_VOCAB}
vocab = get_action('vocabulary_create')(context, data)
log.info("Adding tag %s to vocab %s" % ('jazz', forms.GENRE_VOCAB))
data = {'name': 'jazz', 'vocabulary_id': vocab['id']}
get_action('tag_create')(context, data)
log.info("Adding tag %s to vocab %s" % ('soul', forms.GENRE_VOCAB))
data = {'name': 'soul', 'vocabulary_id': vocab['id']}
get_action('tag_create')(context, data)
def old_unresolved(org, days=30):
q = _issue_query(org, False, days=days)
return model.Session.execute(q).scalar()
def get_resources(state='active', publisher_ref=None, resource_id=None, dataset_name=None):
''' Returns all active resources, or filtered by the given criteria. '''
from ckan import model
resources = model.Session.query(model.Resource) \
.filter_by(state=state)
if hasattr(model, 'ResourceGroup'):
# earlier CKANs had ResourceGroup
resources = resources.join(model.ResourceGroup)
resources = resources \
.join(model.Package) \
.filter_by(state='active')
criteria = [state]
if publisher_ref:
publisher = model.Group.get(publisher_ref)
assert publisher
resources = resources.filter(model.Package.owner_org == publisher.id)
criteria.append('Publisher:%s' % publisher.name)
if dataset_name:
resources = resources.filter(model.Package.name == dataset_name)
criteria.append('Dataset:%s' % dataset_name)