Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def decorated(*args, **kwargs):
args, base_url = args[:-1], args[-1]
Session.remove()
rev = repo.new_revision()
package = Package(name=TEST_PACKAGE_NAME)
Session.add(package)
resources = [
Resource(
description=u'Resource #%d' % (ix,),
url=(base_url + url).decode('ascii')
)
for ix, url in enumerate(resource_urls)
]
for r in resources:
Session.add(r)
package.resources.append(r)
repo.commit()
context = {
'model': model, 'session': model.Session, 'id': package.id
}
package_dict = package_dictize(package, context)
try:
self.log = logging.getLogger(__name__)
import ckan.model as model
model.Session.remove()
model.Session.configure(bind=model.meta.engine)
model.repo.new_revision()
formats = ['csv', 'xls']
if len(self.args) == 1:
formats = self.args[0].split(',')
log.info("Processing %s" % ' and '.join(formats))
for fmt in formats:
q = model.Session.query(model.Resource)\
.filter(func.lower(model.Resource.format) == func.lower(fmt))\
.filter(model.Resource.state == 'active')
total = q.count()
records = q.order_by(func.random()).limit(self.options.count).all()
self.log.info("We have %d records from %d files of %s format" %
(len(records), total, fmt))
self.log.info("=" * 50)
success_count, fail_count = 0, 0
for r in records:
t0 = time.time()
success, msg = self._test_resource(r)
duration = time.time() - t0
if success:
self.log.info(" OK (%0.2fs) - %s" % (duration, r.id))
def fetch_recent_package_activity_list_html(
context, user_id=None, user_id_not=None, only_privatized=False,
only_resourceful=False, limit=30):
# Fetch recent revisions, store as list oredered by time
recent_revisions_query = model.Session.query(model.PackageRevision).distinct()
if only_resourceful:
recent_revisions_query = (
recent_revisions_query
.join(model.Resource, model.Resource.package_id == model.PackageRevision.id)
.filter(model.Resource.state == "active"))
if user_id is not None:
recent_revisions_query = recent_revisions_query.filter(
model.PackageRevision.creator_user_id == user_id)
if user_id_not is not None:
recent_revisions_query = recent_revisions_query.filter(
model.PackageRevision.creator_user_id != user_id_not)
if only_privatized:
recent_revisions_query = recent_revisions_query.filter(
model.PackageRevision.private)
recent_revisions_query = (
recent_revisions_query
.order_by(model.PackageRevision.revision_timestamp.desc())
.limit(limit))
recent_revisions = [r for r in recent_revisions_query]
def fetch_recent_package_activity_list_html(
context, user_id=None, user_id_not=None, only_privatized=False,
only_resourceful=False, limit=30):
# Fetch recent revisions, store as list oredered by time
recent_revisions_query = model.Session.query(model.PackageRevision).distinct()
if only_resourceful:
recent_revisions_query = (
recent_revisions_query
.join(model.Resource, model.Resource.package_id == model.PackageRevision.id)
.filter(model.Resource.state == "active"))
if user_id is not None:
recent_revisions_query = recent_revisions_query.filter(
model.PackageRevision.creator_user_id == user_id)
if user_id_not is not None:
recent_revisions_query = recent_revisions_query.filter(
model.PackageRevision.creator_user_id != user_id_not)
if only_privatized:
recent_revisions_query = recent_revisions_query.filter(
model.PackageRevision.private)
recent_revisions_query = (
recent_revisions_query
.order_by(model.PackageRevision.revision_timestamp.desc())
.limit(limit))
recent_revisions = [r for r in recent_revisions_query]
def _sync_resources(self):
queue = model.Session.query(Extras).join(model.Package).join(
model.Resource).filter(model.Resource.url_type == None)
for record in queue:
try:
creds = record.package.get_groups(
'organization').pop().datadotworld_credentials
except Exception as e:
print(e)
continue
api = API(creds.owner, creds.key)
api.sync_resources(record.id)
# earlier CKANs had ResourceGroup
resources = resources.join(model.ResourceGroup)
resources = resources \
.join(model.Package) \
.filter_by(state='active')
criteria = [state]
if publisher_ref:
publisher = model.Group.get(publisher_ref)
assert publisher
resources = resources.filter(model.Package.owner_org == publisher.id)
criteria.append('Publisher:%s' % publisher.name)
if dataset_name:
resources = resources.filter(model.Package.name == dataset_name)
criteria.append('Dataset:%s' % dataset_name)
if resource_id:
resources = resources.filter(model.Resource.id == resource_id)
criteria.append('Resource:%s' % resource_id)
resources = resources.all()
print '%i resources (%s)' % (len(resources), ' '.join(criteria))
return resources
http://lasigpublic.nerc-lancaster.ac.uk/ArcGIS/services/Biodiversity/GMFarmEvaluation/MapServer/WMSServer?LAYERS=WinterOilseedRape&QUERY_LAYERS=WinterOilseedRape&STYLES=&SERVICE=WMS&VERSION=1.1.1&REQUEST=GetFeatureInfo&EXCEPTIONS=application%2Fvnd.ogc.se_xml&BBOX=-1.628338%2C52.686046%2C-0.086204%2C54.8153&FEATURE_COUNT=11&HEIGHT=845&WIDTH=612&FORMAT=image%2Fpng&INFO_FORMAT=application%2Fvnd.ogc.wms_xml&SRS=EPSG%3A4258&X=327&Y=429
'''
# avoid status_code_redirect intercepting error responses
request.environ['pylons.status_code_redirect'] = False
wms_url = request.params.get('url')
# Check parameter
if not (wms_url):
response.status_int = 400
return 'Missing url parameter'
# Check base of URL is in CKAN (otherwise we are an open proxy)
# (the parameters get changed by the Preview widget)
base_wms_url = get_wms_base_url(wms_url)
query = model.Session.query(model.Resource).filter(model.Resource.url.like(base_wms_url+'%'))
if query.count() == 0:
# Try in the 'wms_base_urls' extras too, as some WMSs use different
# bases (specified in their GetCapabilities response)
model_attr = getattr(model.Resource, 'extras')
field = 'wms_base_urls'
term = base_wms_url #.replace('/', '\\/').replace(':', '\\:')
like = sqlalchemy.or_(
model_attr.ilike(u'''%%"%s": "%%%s%%",%%''' % (field, term)),
model_attr.ilike(u'''%%"%s": "%%%s%%"}''' % (field, term))
)
q = model.Session.query(model.Resource).filter(like)
if q.count() == 0:
response.status_int = 403
return 'Base of WMS URL not known: %r' % base_wms_url
content = self._read_url(wms_url)
def get_for_package(cls, package_id):
'''Returns the QA for the given package. May not be any if the package
has no resources or has not been archived. It checks the resources are
not deleted.'''
return model.Session.query(cls) \
.filter(cls.package_id == package_id) \
.join(model.Resource, cls.resource_id == model.Resource.id) \
.filter(model.Resource.state == 'active') \
.all()
def update_resource_(resource_id):
from ckan import model
resource = model.Resource.get(resource_id)
if not resource:
raise QAError('Resource ID not found: %s' % resource_id)
qa_result = resource_score(resource)
log.info('Openness scoring: \n%r\n%r\n%r\n\n', qa_result, resource,
resource.url)
save_qa_result(resource, qa_result)
log.info('CKAN updated with openness score')
if toolkit.check_ckan_version(max_version='2.2.99'):
package = resource.resource_group.package
else:
package = resource.package
if package:
# Refresh the index for this dataset, so that it contains the latest
# qa info
_update_search_index(package.id)
from ckanext.archiver.model import Archival
counts = {}
# Get all the broken datasets and build up the results by org
orgs = model.Session.query(model.Group)\
.filter(model.Group.type == 'organization')\
.filter(model.Group.state == 'active').all()
for org in add_progress_bar(
orgs, 'Part 1/2' if include_sub_organizations else None):
archivals = (model.Session.query(Archival)
.filter(Archival.is_broken == True) # noqa
.join(model.Package, Archival.package_id == model.Package.id)
.filter(model.Package.owner_org == org.id)
.filter(model.Package.state == 'active')
.join(model.Resource, Archival.resource_id == model.Resource.id)
.filter(model.Resource.state == 'active'))
broken_resources = archivals.count()
broken_datasets = archivals.distinct(model.Package.id).count()
num_datasets = model.Session.query(model.Package)\
.filter_by(owner_org=org.id)\
.filter_by(state='active')\
.count()
num_resources = model.Session.query(model.Package)\
.filter_by(owner_org=org.id)\
.filter_by(state='active')
if p.toolkit.check_ckan_version(max_version='2.2.99'):
num_resources = num_resources.join(model.ResourceGroup)
num_resources = num_resources \
.join(model.Resource)\
.filter_by(state='active')\
.count()