Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
it('generates EMS product metadata reports through the Cumulus API', async () => {
const collection = { name: 'A2_SI25_NRT', version: '0' };
const inputPayload = {
reportType: 'metadata',
startTime: moment.utc().subtract(1, 'days').startOf('day').format(),
endTime: moment.utc().add(1, 'days').startOf('day').format(),
collectionId: constructCollectionId(collection.name, collection.version),
invocationType: 'RequestResponse'
};
const response = await emsApi.createEmsReports({
prefix: config.stackName,
request: inputPayload
});
// verify the report has the specified collection
const reports = JSON.parse(response.body).reports;
expect(reports.length).toEqual(1);
const parsed = parseS3Uri(reports[0].file);
expect(await fileExists(parsed.Bucket, parsed.Key)).not.toBe(false);
const obj = await getS3Object(parsed.Bucket, parsed.Key);
const reportRecords = obj.Body.toString().split('\n');
describe('The Sync Granules workflow', () => {
const testId = createTimestampedTestId(config.stackName, 'SyncGranuleSuccess');
const testSuffix = createTestSuffix(testId);
const testDataFolder = createTestDataPath(testId);
const inputPayloadFilename = './spec/parallel/syncGranule/SyncGranule.input.payload.json';
const providersDir = './data/providers/s3/';
const collectionsDir = './data/collections/s3_MOD09GQ_006';
const collection = { name: `MOD09GQ${testSuffix}`, version: '006' };
const provider = { id: `s3_provider${testSuffix}` };
const newCollectionId = constructCollectionId(collection.name, collection.version);
let inputPayload;
let expectedPayload;
let expectedS3TagSet;
let workflowExecution;
process.env.ExecutionsTable = `${config.stackName}-ExecutionsTable`;
const executionModel = new Execution();
process.env.CollectionsTable = `${config.stackName}-CollectionsTable`;
const collectionModel = new Collection();
beforeAll(async () => {
// populate collections, providers and test data
await Promise.all([
uploadTestDataToBucket(config.bucket, s3data, testDataFolder),
addCollections(config.stackName, config.bucket, collectionsDir, testSuffix),
process.env.cmr_provider, process.env.cmr_client_id, 'collections', [], 'echo10'
);
let nextCmrItem = await cmrCollectionsIterator.peek();
while (nextCmrItem) {
await cmrCollectionsIterator.shift(); // eslint-disable-line no-await-in-loop
// evaluate each EMS field based on CMR collection and build EMS record
const emsRecord = Object.entries(emsMapping)
// eslint-disable-next-line no-loop-func
.map(([field, value]) =>
({ [field]: (isFunction(value) ? value(nextCmrItem.Collection) : value) }))
.reduce((returnObj, currentValue) => ({ ...returnObj, ...currentValue }), {});
const collectionId = constructCollectionId(
nextCmrItem.Collection.ShortName, nextCmrItem.Collection.VersionId
);
const lastUpdate = nextCmrItem.Collection.LastUpdate || nextCmrItem.Collection.InsertTime;
collections.push({ collectionId, lastUpdate, emsRecord });
nextCmrItem = await cmrCollectionsIterator.peek(); // eslint-disable-line no-await-in-loop
}
return collections;
}
describe('The Sync Granules workflow', () => {
const testId = createTimestampedTestId(config.stackName, 'SyncGranuleSuccess');
const testSuffix = createTestSuffix(testId);
const testDataFolder = createTestDataPath(testId);
const inputPayloadFilename = './spec/parallel/syncGranule/SyncGranule.input.payload.json';
const providersDir = './data/providers/s3/';
const collectionsDir = './data/collections/s3_MOD09GQ_006';
const collection = { name: `MOD09GQ${testSuffix}`, version: '006' };
const provider = { id: `s3_provider${testSuffix}` };
const newCollectionId = constructCollectionId(collection.name, collection.version);
let inputPayload;
let expectedPayload;
let expectedS3TagSet;
let workflowExecution;
process.env.ExecutionsTable = `${config.stackName}-ExecutionsTable`;
const executionModel = new Execution();
process.env.CollectionsTable = `${config.stackName}-CollectionsTable`;
const collectionModel = new Collection();
beforeAll(async () => {
// populate collections, providers and test data
await Promise.all([
uploadTestDataToBucket(config.bucket, s3data, testDataFolder),
addCollections(config.stackName, config.bucket, collectionsDir, testSuffix),
this.host = this.provider.host;
this.username = this.provider.username;
this.password = this.provider.password;
this.checksumFiles = {};
this.supportedChecksumFileTypes = ['md5', 'cksum', 'sha1', 'sha256'];
this.forceDownload = forceDownload;
if (fileStagingDir && fileStagingDir[0] === '/') this.fileStagingDir = fileStagingDir.substr(1);
else this.fileStagingDir = fileStagingDir;
this.duplicateHandling = duplicateHandling;
// default collectionId, could be overwritten by granule's collection information
if (this.collection) {
this.collectionId = constructCollectionId(
this.collection.dataType || this.collection.name, this.collection.version
);
}
}
it('generates a report showing collections that are in the Cumulus but not in CMR', () => {
const extraCollection = constructCollectionId(extraCumulusCollection.name.S, extraCumulusCollection.version.S);
expect(report.collectionsInCumulusCmr.onlyInCumulus).toContain(extraCollection);
expect(report.collectionsInCumulusCmr.onlyInCumulus).not.toContain(collectionId);
});
generateDocFromPayload(payload) {
const name = get(payload, 'cumulus_meta.execution_name');
const arn = aws.getExecutionArn(
get(payload, 'cumulus_meta.state_machine'),
name
);
if (!arn) {
throw new Error('State Machine Arn is missing. Must be included in the cumulus_meta');
}
const execution = aws.getExecutionUrl(arn);
const collectionId = constructCollectionId(
get(payload, 'meta.collection.name'), get(payload, 'meta.collection.version')
);
const doc = {
name,
arn,
parentArn: get(payload, 'cumulus_meta.parentExecutionArn'),
execution,
tasks: get(payload, 'meta.workflow_tasks'),
error: parseException(payload.exception),
type: get(payload, 'meta.workflow_name'),
collectionId: collectionId,
status: get(payload, 'meta.status', 'unknown'),
createdAt: get(payload, 'cumulus_meta.workflow_start_time'),
timestamp: Date.now()
};
createPdrFromSns(payload) {
const name = get(payload, 'cumulus_meta.execution_name');
const pdrObj = get(payload, 'payload.pdr', get(payload, 'meta.pdr'));
const pdrName = get(pdrObj, 'name');
if (!pdrName) return Promise.resolve();
const arn = aws.getExecutionArn(
get(payload, 'cumulus_meta.state_machine'),
name
);
const execution = aws.getExecutionUrl(arn);
const collection = get(payload, 'meta.collection');
const collectionId = constructCollectionId(collection.name, collection.version);
const stats = {
processing: get(payload, 'payload.running', []).length,
completed: get(payload, 'payload.completed', []).length,
failed: get(payload, 'payload.failed', []).length
};
stats.total = stats.processing + stats.completed + stats.failed;
let progress = 0;
if (stats.processing > 0 && stats.total > 0) {
progress = ((stats.total - stats.processing) / stats.total) * 100;
}
else if (stats.processing === 0 && stats.total > 0) {
progress = 100;
}
const dbCollectionIds = dbCollectionsItems.map((item) =>
constructCollectionId(item.name, item.version)).sort();
describe('The S3 Ingest Granules workflow configured to ingest UMM-G', () => {
const testId = createTimestampedTestId(config.stackName, 'IngestUMMGSuccess');
const testSuffix = createTestSuffix(testId);
const testDataFolder = createTestDataPath(testId);
const inputPayloadFilename = './spec/parallel/ingestGranule/IngestGranule.input.payload.json';
const providersDir = './data/providers/s3/';
const collectionsDir = './data/collections/s3_MOD09GQ_006-umm';
const collection = { name: `MOD09GQ${testSuffix}`, version: '006' };
const provider = { id: `s3_provider${testSuffix}` };
const newCollectionId = constructCollectionId(collection.name, collection.version);
let workflowExecution = null;
let inputPayload;
let expectedPayload;
let postToCmrOutput;
let granule;
let server;
process.env.AccessTokensTable = `${config.stackName}-AccessTokensTable`;
const accessTokensModel = new AccessToken();
process.env.GranulesTable = `${config.stackName}-GranulesTable`;
process.env.ExecutionsTable = `${config.stackName}-ExecutionsTable`;
const executionModel = new Execution();
process.env.CollectionsTable = `${config.stackName}-CollectionsTable`;
const collectionModel = new Collection();
process.env.ProvidersTable = `${config.stackName}-ProvidersTable`;