Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
const steps = resources.map(async res => {
if (excelFormats.includes(res.descriptor.format)) {
const buffer = await res.buffer
const workbook = XLSX.read(buffer, {type: 'buffer', bookSheets: true})
if (sheets === 'all') {
sheets = Array.from(Array(workbook.SheetNames.length).keys())
// We are using sheet idx starting from 1 so we need to add 1 to each idx:
sheets = sheets.map(value => value + 1)
} else if (sheets) { // When sheets are indexes of sheets separated by ','
sheets = sheets.split(',').map(value => parseInt(value))
} else { // Default case
sheets = [1]
}
for (let i of sheets) {
const rows = await toArray(await xlsxParser(res, false, i-1))
const schema = await infer(rows)
const step = {
input: res.descriptor.name,
output: `${res.descriptor.name}-sheet-${i}`,
tabulator: {
sheet: i
},
schema
}
processingSteps.push(step)
}
}
})
await Promise.all(steps)
async validateData(descriptor, absPath) {
// TODO: handle inlined data resources
let options = {schema: descriptor.schema}
if (descriptor.dialect) {
Object.assign(options, descriptor.dialect)
}
const table = await Table.load(absPath, options)
try {
await table.read()
} catch (err) {
err.resource = descriptor.name
err.path = absPath
throw err
}
return true
}
async function validateData(schema, absPath) {
// TODO: handle inlined data resources
const table = await Table.load(absPath, {schema})
await table.read()
return true
}
newReference: this.state.newReference,
newRefReference: this.state.newRefReference
}
// TODO: hardcoded
if (resourceDescriptor.name === 'currencies' && !this.state.newReference) {
const source = [keys(payload.newResource.data[0])].concat([values(payload.newResource.data[0])]);
const relations = {
exchange_rates: referenceTables['exchange_rates']
};
let table;
const prefixedValues = {
"modified_currency": this.state.newResource['modified_currency'].value
}
try {
table = await Table.load(source, {schema});
const rows = await table.read({forceCast: true, relations});
const errors = rows.filter((row) => row.errors);
if (errors.length) {
this.setState({
prefixedValues,
resourceValid: {
valid: false,
message: errors[0].errors[0].errors[0].message
}
});
} else {
this.setState({
resourceValid: {valid: true}
})
this.props.onAddNew(payload)
}
try {
const tableLength = source.length;
const chunk = DEFAULT_CHUNK_SIZE;
let i = 0
let errors = []
for(i; i < tableLength; i += chunk) {
dispatch({
type: VALIDATE_TABLE_REQUEST,
payload: {
status: 'loading',
loader: `validating ${i} rows`
}
})
const offset = i / chunk
const chunkTable = [source[0]].concat(source.slice(i+1-offset, i+chunk-offset))
const table = await Table.load(chunkTable, {schema});
const rows = await table.read({forceCast: true, relations});
const chunkErrors = rows.filter((row) => row.errors)
if (chunkErrors.length) {
chunkErrors.forEach((error) => {
error.rowNumber = error.rowNumber + chunk * offset - offset
});
errors = errors.concat(chunkErrors)
}
}
// const table = await Table.load(source, {schema});
// const rows = await table.read({forceCast: true});
// const errors = rows.filter((row) => row.errors)
if (errors.length) {
dispatch({
type: VALIDATE_TABLE_FAILURE,
payload: {
var _ref2 = (0, _asyncToGenerator3.default)(function* (schema, absPath) {
// TODO: handle inlined data resources
const table = yield Table.load(absPath, { schema });
yield table.read();
return true;
});
async addSchema() {
// Ensure resource is tabular
if (knownTabularFormats.indexOf(this.descriptor.format) === -1) {
throw new Error('File is not in known tabular format.')
}
const rows = await toArray(await this.rows())
this.descriptor.schema = await infer(rows)
}
}
return (0, _asyncToGenerator3.default)(function* () {
// Ensure resource is tabular
if (knownTabularFormats.indexOf(_this2.descriptor.format) === -1) {
throw new Error('File is not in known tabular format.');
}
const rows = yield toArray((yield _this2.rows()));
_this2.descriptor.schema = yield infer(rows);
})();
}
this.descriptor.schema = await infer(this.descriptor.data);
return;
} // Get parserOptions so we can use it when "infering" schema:
const parserOptions = await guessParseOptions(this); // We also need to include parserOptions in "dialect" property of descriptor:
this.descriptor.dialect = {
delimiter: parserOptions.delimiter,
quoteChar: parserOptions.quote // Now let's get a stream from file and infer schema:
};
let thisFileStream = await this.stream({
size: 100
});
this.descriptor.schema = await infer(thisFileStream, parserOptions);
}
async addSchema() {
// Ensure file is tabular
if (knownTabularFormats.indexOf(this.descriptor.format) === -1) {
throw new Error('File is not in known tabular format.')
}
if (this.displayName === 'FileInline') {
this.descriptor.schema = await infer(this.descriptor.data)
return
}
// Get parserOptions so we can use it when "infering" schema:
const parserOptions = await guessParseOptions(this)
// We also need to include parserOptions in "dialect" property of descriptor:
this.descriptor.dialect = {
delimiter: parserOptions.delimiter,
quoteChar: parserOptions.quote
}
// Now let's get a stream from file and infer schema:
let thisFileStream = await this.stream({size: 100})
this.descriptor.schema = await infer(thisFileStream, parserOptions)
}
}