Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
export async function mapBucket(Bucket: string, keyFilter: (key: string) => boolean) {
const faastModule = await faast("aws", m, {
memorySize: 2048,
timeout: 300,
mode: "queue",
concurrency: 2000,
childProcess: true,
gc: "off"
// awsLambdaOptions: { TracingConfig: { Mode: "Active" } }
});
console.log(`Logs: ${faastModule.logUrl()} `);
faastModule.on("stats", s => {
console.log(`${s}`);
});
const bandwidth = new Statistics();
try {
let allObjects = await listAllObjects(Bucket);
allObjects = allObjects.filter(obj => keyFilter(obj.Key!));
const promises = [];
console.log(`Bucket ${Bucket} contains ${allObjects.length} matching objects`);
const start = Date.now();
for (const Obj of allObjects) {
promises.push(
faastModule.functions
.processBucketObject(Bucket, Obj.Key!)
.catch((err: FaastError) => {
console.log(`Error processing ${Obj.Key!}`);
console.log(`Logs: ${err.logUrl}`);
})
);
const workload = (Bucket: string, filter: FilterFn) => async (
faastModule: FaastModule
) => {
const remote = faastModule.functions;
let allObjects = await listAllObjects(Bucket);
allObjects = allObjects.filter(obj => filter(obj.Key!));
const promises = [];
const start = Date.now();
for (const Obj of allObjects) {
promises.push(remote.processBucketObject(Bucket, Obj.Key!));
break;
}
const results = await Promise.all(promises);
const elapsed = Date.now() - start;
let bytes = 0;
const bandwidth = new Statistics();
for (const result of results) {
if (!result) {
continue;
}
bytes += result.bytes;
bandwidth.update(result.bandwidthMbps);
}
const metrics: BandwidthMetrics = {
bytesGB: bytes / GB,
bandwidthMbps: bandwidth.mean
};
return metrics;
};