Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import parallelLimit from 'async/parallelLimit';
import Bottleneck from 'bottleneck';
import { unifyMeasurementUnits } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
// Default rate limiting on API is set to 5 requests/sec.
// > Please send an email to Tools.support@epa.vic.gov.au with subject
// > 'EPA API Access Request: Increase rate-limiting' and a justified
// > reason if you want to get it increased for your subscriptions.
var maxRequestsPerSecond = 5;
var limiter = new Bottleneck({
reservoir: maxRequestsPerSecond, // allow 5 requests
reservoirRefreshAmount: maxRequestsPerSecond,
reservoirRefreshInterval: 1000, // every 1000ms
maxConcurrent: 1,
minTime: (1000 / maxRequestsPerSecond) + 50 // to stagger requests out throught each second adding a 50ms buffer
});
export const name = 'victoria';
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import { parallel } from 'async';
import { flattenDeep } from 'lodash';
import { acceptableParameters, convertUnits } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
export const name = 'buenos-aires';
const timezone = 'America/Argentina/Buenos_Aires';
export function fetchData (source, callback) {
request.get(source.url, (err, res, body) => {
if (err || res.statusCode !== 200) {
return callback({message: 'Failed to load entry point url'}, null);
}
let tasks = [];
let $ = cheerio.load(body);
const stations = $('#estacion option').filter(function (i, el) {
// skip non working station
return $(this).text() !== 'PALERMO';
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import { acceptableParameters } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
exports.name = 'sweden';
exports.fetchData = function (source, cb) {
request(source.url, function (err, res, body) {
if (err || res.statusCode !== 200) {
return cb({message: 'Failure to load data url.'});
}
// Wrap everything in a try/catch in case something goes wrong
try {
// Format the data
var data = formatData(body);
// Make sure the data is valid
if (data === undefined) {
/**
* This code is responsible for implementing all methods related to fetching
* and returning data from StateAir.net data source.
*/
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { cloneDeep } from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
export const name = 'beijing';
/**
* Fetches the data for a given source and returns an appropriate object
* @param {object} source A valid source object
* @param {function} cb A callback of the form cb(err, data)
*/
export const fetchData = function (source, cb) {
request(source.url, function (err, res, body) {
if (err || res.statusCode !== 200) {
return cb({message: 'Failure to load data url.'});
}
// Wrap everything in a try/catch in case something goes wrong
try {
/**
* This code is responsible for implementing all methods related to fetching
* and returning data for the Norwegian data sources.
*/
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { default as moment } from 'moment-timezone';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
exports.name = 'norway';
/**
* Fetches the data for a given source and returns an appropriate object
* @param {object} source A valid source object
* @param {function} cb A callback of the form cb(err, data)
*/
exports.fetchData = function (source, cb) {
request(source.url, function (err, res, body) {
if (err || res.statusCode !== 200) {
return cb({message: 'Failure to load data url.'});
}
// Wrap everything in a try/catch in case something goes wrong
try {
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { default as moment } from 'moment-timezone';
import { convertUnits, safeParse, acceptableParameters } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
export const name = 'data_gov_in';
export function fetchData (source, cb) {
// Load initial results set and grab more if needed recursively
let offset = 0;
const limit = 1000;
const url = `${source.url}?api-key=${process.env.DATA_GOV_IN_TOKEN}&limit=${limit}&fields=city,station,last_update,pollutant_id,pollutant_avg&format=json`;
let results = [];
const getResults = function (url) {
return getData(url, (err, body) => {
if (err) {
return cb({message: err.message});
}
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import cheerio from 'cheerio';
import { default as moment } from 'moment-timezone';
import { filter, flattenDeep, isFinite } from 'lodash';
import { parallel } from 'async';
import { acceptableParameters, convertUnits } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
export const name = 'cpcb';
export function fetchData (source, cb) {
// Load initial page to get individual states
request(source.url, (err, res, body) => {
if (err || res.statusCode !== 200) {
return cb({message: 'Failure to load data url.'});
}
const $ = cheerio.load(body);
// Loop over each state and add tasks
let tasks = [];
const states = grabActive($);
states.forEach((e, i) => {
tasks.push(handleState($(e)));
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { cloneDeep } from 'lodash';
import { default as moment } from 'moment-timezone';
// note: this is the 'synchronous' version (lost hours to this!)
import { default as parse } from 'csv-parse/lib/sync';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
exports.name = 'au_sa';
exports.fetchData = function (source, cb) {
// Fetch the data
request(source.url, function (err, res, body) {
if (err || res.statusCode !== 200) {
return cb({message: 'Failure to load data url.'});
}
// Wrap everything in a try/catch in case something goes wrong
try {
// Format the data
var data = formatData(body, source);
if (data === undefined) {
return cb({message: 'Failure to parse data.'});
}
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import async from 'async';
import { removeUnwantedParameters } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
exports.name = 'netherlands';
exports.fetchData = function (source, cb) {
var finalURL = source.url;
request(finalURL, function (err, res, body) {
if (err || res.statusCode !== 200) {
return cb({message: 'Failure to load data url.'});
}
// Fetch list with available files from server
var fileList = listApachetree(body, finalURL);
// Filter list so it only contains the most recent group of files.
var recentDate = _.last(fileList).date.getTime();
var recentFiles = _.filter(fileList, function (f) {
/**
* This code is responsible for implementing all methods related to
* fetching and returning data for the Indian data sources.
*/
'use strict';
import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import log from '../lib/logger';
import { removeUnwantedParameters, convertUnits } from '../lib/utils';
exports.name = 'india';
/**
* Fetches the data for a given source and returns an appropriate object
* @param {object} source A valid source object
* @param {function} cb A callback of the form cb(err, data)
*/
exports.fetchData = function (source, cb) {
request(source.url, function (err, res, body) {
if (err || res.statusCode !== 200) {