Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
token = tuple.token;
tag = tuple.tag;
} else {
// PADDING
token = TAGS[TAGS.PAD_IDX];
tag = TAGS[TAGS.PAD_IDX];
}
// Note that we reuse the tensors for a given token or tag.
const tokenEmbedding = tokenEmbeddings.get(token);
const tagOnehot = labelOneHots.get(tag);
exampleX.push(tokenEmbedding);
exampleY.push(tagOnehot);
tf.util.assert(
tokenEmbedding != null,
() => console.log(`Error getting token embedding for ${token}`));
tf.util.assertShapesMatch(
tokenEmbedding.shape, [EMBEDDING_SIZE],
() => console.log(`Wrong shape for token embedding of ${token}`));
tf.util.assert(
tagOnehot != null,
() => console.log(`Error getting label onehot for ${tag}`));
tf.util.assertShapesMatch(
tagOnehot.shape, [TAGS.length],
() => console.log(`Wrong shape for label onehot for ${tag}`));
}
export function getInputTensorFromFrequencyData(
freqData: Float32Array, shape: number[]): tf.Tensor {
const vals = new Float32Array(tf.util.sizeFromShape(shape));
// If the data is less than the output shape, the rest is padded with zeros.
vals.set(freqData, vals.length - freqData.length);
return tf.tensor(vals, shape);
}
Number.isInteger(snippetLength) && snippetLength > 0,
() =>
`snippetLength must be a positive integer, but got ${snippetLength}`);
if (focusIndex != null) {
tf.util.assert(
Number.isInteger(focusIndex) && focusIndex >= 0,
() =>
`focusIndex must be a non-negative integer, but got ${focusIndex}`);
}
tf.util.assert(
Number.isInteger(windowLength) && windowLength > 0,
() => `windowLength must be a positive integer, but got ${windowLength}`);
tf.util.assert(
Number.isInteger(windowHop) && windowHop > 0,
() => `windowHop must be a positive integer, but got ${windowHop}`);
tf.util.assert(
windowLength <= snippetLength,
() => `windowLength (${windowLength}) exceeds snippetLength ` +
`(${snippetLength})`);
tf.util.assert(
focusIndex < snippetLength,
() => `focusIndex (${focusIndex}) equals or exceeds snippetLength ` +
`(${snippetLength})`);
if (windowLength === snippetLength) {
return [[0, snippetLength]];
}
const windows: Array<[number, number]> = [];
if (focusIndex == null) {
// Deal with the special case of no focus frame:
function convertToTensors(data, targets, testSplit) {
const numExamples = data.length;
if (numExamples !== targets.length) {
throw new Error('data and split have different numbers of examples');
}
// Randomly shuffle `data` and `targets`.
const indices = [];
for (let i = 0; i < numExamples; ++i) {
indices.push(i);
}
tf.util.shuffle(indices);
const shuffledData = [];
const shuffledTargets = [];
for (let i = 0; i < numExamples; ++i) {
shuffledData.push(data[indices[i]]);
shuffledTargets.push(targets[indices[i]]);
}
// Split the data into a training set and a tet set, based on `testSplit`.
const numTestExamples = Math.round(numExamples * testSplit);
const numTrainExamples = numExamples - numTestExamples;
const xDims = shuffledData[0].length;
const xs = tf.tensor2d(shuffledData, [numExamples, xDims]);
export function generateDataForTraining(trainSplit = 0.25, valSplit = 0.15) {
tf.util.assert(
trainSplit > 0 && valSplit > 0 && trainSplit + valSplit <= 1,
`Invalid trainSplit (${trainSplit}) and valSplit (${valSplit})`);
const dateTuples = [];
const MIN_YEAR = 1950;
const MAX_YEAR = 2050;
for (let date = new Date(MIN_YEAR,0,1);
date.getFullYear() < MAX_YEAR;
date.setDate(date.getDate() + 1)) {
dateTuples.push([date.getFullYear(), date.getMonth() + 1, date.getDate()]);
}
tf.util.shuffle(dateTuples);
const numTrain = Math.floor(dateTuples.length * trainSplit);
const numVal = Math.floor(dateTuples.length * valSplit);
console.log(`Number of dates used for training: ${numTrain}`);
console.log(`Number of dates used for validation: ${numVal}`);
console.log(
`Number of dates used for testing: ` +
`${dateTuples.length - numTrain - numVal}`);
function dateTuplesToTensor(dateTuples) {
return tf.tidy(() => {
const inputs =
dateFormat.INPUT_FNS.map(fn => dateTuples.map(tuple => fn(tuple)));
const inputStrings = [];
inputs.forEach(inputs => inputStrings.push(...inputs));
const encoderInput =
async function run(path) {
const image = readImageAsJpeg(path);
const input = imageToInput(image, NUMBER_OF_CHANNELS);
console.log(' - Loading model...');
let start = tf.util.now();
const model = await mobilenet.load();
let end = tf.util.now();
console.log(` - Mobilenet load: ${end - start}ms`);
start = tf.util.now();
console.log(' - Coldstarting model...');
await model.classify(input);
end = tf.util.now();
console.log(` - Mobilenet cold start: ${end - start}ms`);
const times = 100;
let totalMs = 0;
console.log(` - Running inference (${times}x) ...`);
for (let i = 0; i < times; i++) {
start = tf.util.now();
await model.classify(input);
end = tf.util.now();
constructor(dataIdentifier, textString, sampleLen, sampleStep) {
tf.util.assert(
sampleLen > 0,
`Expected sampleLen to be a positive integer, but got ${sampleLen}`);
tf.util.assert(
sampleStep > 0,
`Expected sampleStep to be a positive integer, but got ${sampleStep}`);
if (!dataIdentifier) {
throw new Error('Model identifier is not provided.');
}
this.dataIdentifier_ = dataIdentifier;
this.textString_ = textString;
this.textLen_ = textString.length;
this.sampleLen_ = sampleLen;
this.sampleStep_ = sampleStep;
this.getCharSet_();
this.convertAllTextToIndices_();
onTrainBegin: async (logs) => {
beginMillis = tf.util.now();
status(
'Please wait and do NOT click anything while the model retrains...',
'blue');
trainProg.value = 0;
},
onTrainEnd: async (logs) => {
setExampleKeyFrameIndex(uid: string, keyFrameIndex: number) {
if (!(uid in this.examples)) {
throw new Error(`Nonexistent example UID: ${uid}`);
}
const spectrogram = this.examples[uid].spectrogram;
const numFrames = spectrogram.data.length / spectrogram.frameSize;
tf.util.assert(
keyFrameIndex >= 0 && keyFrameIndex < numFrames &&
Number.isInteger(keyFrameIndex),
() => `Invalid keyFrameIndex: ${keyFrameIndex}. ` +
`Must be >= 0, < ${numFrames}, and an integer.`);
spectrogram.keyFrameIndex = keyFrameIndex;
}