Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
}, function (err, res, body) {
if(err) {
next(err);
return;
}
var redirect = _.first(_.flattenDeep(_.compact(_.pluck(body, 'http://dbpedia.org/ontology/wikiPageRedirects'))));
if(followRedirection && redirect && redirect.value && level < 1) {
var link = redirect.value.split('/').pop();
if(options.link == link) {
// no need to scrape again...
next(null, body);
return
}
console.log(clc.blackBright('following redirection, level'), clc.cyan(level), clc.blackBright('link'), clc.cyan(link))
setTimeout(function(){
module.exports.dbpedia({
link: link,
level: level + 1
}, next);
}, 2300);
return;
};
next(null, body)
})
},
return function savelink(next){
var resource = _.find(graph.nodes, {slug: n.target}),
entity = _.find(graph.nodes, {slug: n.source});
console.log(clc.blackBright(' - link from:'), entity.slug, clc.blackBright(' --> resource:'), resource.id);
// next()
Entity.create(_.assign({
resource: {
uuid: resource.id
},
username: options.marvin.username
}, entity), function (err, entity) {
if(err){
console.log(err)
return next(err);
}
console.log(clc.blackBright(' - entity', clc.greenBright('saved!'), 'id:'), entity.id, clc.blackBright('- slug:'),entity.props.slug);
next()
});
return through.obj(function (file, encoding, done) {
if(file.isNull()){
console.log('FILE IS NULL')
return done(null, file);
}
console.log(clc.blackBright(' processing:'), file.path);
var graph = JSON.parse(file.contents);
if(!graph || !graph.nodes || !graph.links){
console.log(' not a valid json file:', '' + file.contents);
return done(null, file);
}
console.log(' ',graph.links.length);
// checking for nodes and edges
async.series(graph.nodes.filter(function(n){
return n.mimetype
}).map(function(n, i){
return function saveNode(next){
remove: function(options, callback) {
console.log(clc.yellowBright('\n tasks.helpers.marvin.remove'));
var User = require('../../models/user');
console.log(clc.blackBright(' removing user'), clc.magentaBright('MARVIN'));
User.remove(generator.user.marvin(), function (err) {
if(err)
callback(err);
else
callback(null, options);
});
},
/*
Resource.create(resource, function (err, res) {
if(err) {
q.kill();
callback(err)
} else {
console.log(clc.blackBright(' resource: ', clc.whiteBright(res.id), 'saved,', q.length(), 'resources remaining'));
next();
}
})
}, 1);
q.drain = function() {
var valid_candidates = candidates.filter(function (d) {
return d.type.length > 0
});
console.log(clc.blackBright(' cleaning entities: keeping',clc.whiteBright(valid_candidates.length), 'out of', clc.whiteBright(candidates.length)))
callback(null, resource,valid_candidates );
}
},
}], function(err, result) {
try {
result._ = result._.toUpperCase()
if (result._ === "Y" || !result._) {
finalAnswer = JSON.stringify(finalAnswer);
callback(finalAnswer);
console.log(log.blackBright("\n[*] TCP socket server listening on port " + port));
netcat.listen(port);
} else {
finalAnswer = JSON.stringify(finalAnswer);
callback(finalAnswer);
}
} catch (err) {
console.log("\nLater bro!");
}
})
var q = async.queue(function (node, nextNode) {
console.log(clc.blackBright('entities remaining'), clc.white.bgMagenta(q.length()));
entity.discover(node.id, function(err, res) {
if(err)
throw err;
console.log(res.id, res.name, res.links_wiki, res.languages);
setTimeout(nextNode, 1675);
})
}, 1);
q.push(nodes);
neo4j.save(res, function (err, n) {
if(err)
throw err;
console.log('node', n.id, clc.cyanBright('saved'))
console.log(clc.blackBright('waiting for the next resource ...'))
setTimeout(nextNode, 1675);
})