Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
const Asset = require('../../models/asset');
const scraper = require('../../services/scraper');
const Assets = require('../../services/assets');
const { createLogger } = require('../../services/logging');
const logger = createLogger('jobs:scraper');
const fetch = require('node-fetch');
const { merge } = require('lodash');
const { version } = require('../../package.json');
const { SCRAPER_HEADERS } = require('../../config');
// Load the scraper with the rules.
const metascraper = require('metascraper').load([
require('metascraper-title')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-author')(),
require('metascraper-date')(),
require('./rules/modified')(),
require('./rules/section')(),
]);
let customHeaders = {};
try {
customHeaders = JSON.parse(SCRAPER_HEADERS);
} catch (err) {
console.error('Cannot parse TALK_SCRAPER_HEADERS');
throw err;
}
const fetch = require('node-fetch');
const ProxyAgent = require('proxy-agent');
const { merge } = require('lodash');
const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config');
const kue = require('../kue');
const { version } = require('../../package.json');
// Load the scraper with the rules.
const metascraper = require('metascraper').load([
require('metascraper-title')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-author')(),
require('metascraper-date')(),
require('./rules/modified')(),
require('./rules/section')(),
]);
let customHeaders = {};
try {
customHeaders = JSON.parse(SCRAPER_HEADERS);
} catch (err) {
console.error('Cannot parse TALK_SCRAPER_HEADERS');
throw err;
}
/* eslint-disable no-unused-vars */
const errors = require('@feathersjs/errors');
const mongoose = require('mongoose');
const { URL } = require('url');
const metascraper = require('metascraper').load([
require('metascraper-date')(),
require('metascraper-title')(),
require('metascraper-description')(),
require('metascraper-image')()
]);
const got = require('got');
const Metaphor = require('metaphor');
const engine = new Metaphor.Engine({
preview: false,
tweet: true
});
const getMetadata = async (targetURL, Provider) => {
const data = {
metaphor: {},
// get link metadata
// TODO: add more services and use the metascraper to fill some metadata on the article
const metascraper = require('metascraper').load([
require('metascraper-author')(),
require('metascraper-date')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-logo')(),
require('metascraper-clearbit-logo')(),
require('metascraper-logo-favicon')(),
require('metascraper-publisher')(),
require('metascraper-title')(),
require('metascraper-url')(),
require('metascraper-youtube')(),
]);
const got = require('got');
const _ = require('lodash');
const getMetadata = async (targetUrl, app) => {
const { parse } = require('url')
const { send } = require('micro')
const got = require('got');
const cache = require('memory-cache')
const metascraper = require('metascraper').load([
require('metascraper-author')(),
require('metascraper-date')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-logo')(),
require('metascraper-clearbit-logo')(),
require('metascraper-logo-favicon')(),
require('metascraper-publisher')(),
require('metascraper-title')(),
require('metascraper-url')(),
require('metascraper-logo-favicon')(),
require('metascraper-amazon')(),
require('metascraper-youtube')(),
require('metascraper-soundcloud')(),
require('metascraper-video-provider')()
])