Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async function crawlASite(siteInfo) {
try {
signale.info(`Started crawling site ${siteInfo.address} ${global.tempSiteId[siteInfo.address]}/${global.sitesCount}`)
let dbSchema = await SiteMeta.getDBJson(siteInfo.address),
doc = await DataBase.getSite(siteInfo.address),
siteObj = null,
siteDB = dbSchema ? await SiteMeta.getSiteDataBase(siteInfo.address) : null
if (doc)
siteObj = doc["_source"]
if (!siteObj) { // Site not found, create one
signale.fav(`Discovered a brand new site ${siteInfo.address}`)
siteObj = DataBase.genNewSite(siteInfo) // Init with siteInfo
doc = await DataBase.addSite(siteObj)
} else if ((Date.now() - siteObj.runtime.siteinfo) > (process.env.siteInfoUpdateInterval || 36000000)) { // Update siteInfo
DataBase.setSiteInfo(siteObj, siteInfo)
await DataBase.updateSite(siteObj, doc["_id"])
doc = await DataBase.getSite(siteInfo.address)
}
let modification = { runtime: { feeds: {}, op_files: {} } }
function* promiseGenerator() {
for (let crawler_name in modules) {
if (modules[crawler_name] && modules[crawler_name].prototype instanceof modules["BaseCrawler"]) {
yield (async () => {
try {
let crawler = new modules[crawler_name]({ dbSchema, siteDB, siteObj, address: siteInfo.address, siteId: doc["_id"] })
await crawler.crawl(modification)
} catch (e) {