Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
private *predictInternal(functionDecl: string) {
const { ast: astTokens, comments: commentTokens } = this.tokenizers;
const {
max_ast_len,
max_comment_len,
ast_vocab_size,
comment_vocab_size,
character_tokenizer
} = this.tokenizers.params;
const ast = this.ast(functionDecl);
const astVector = ast
.split(' ')
.slice(0, max_ast_len)
.map((token) => (token in astTokens ? astTokens[token] : astTokens['UNK']));
const inputSeq = tf
.oneHot(astVector, ast_vocab_size)
.pad([[0, max_ast_len - astVector.length], [0, 0]])
.expandDims();
let statesValues = this.encoderModel.predict(inputSeq) as tf.Tensor[];
const startToken = character_tokenizer ? '/' : '';
const startTokenValue = parseInt(
Object.keys(commentTokens).find((k) => commentTokens[k] === startToken)!,
10
);
if (character_tokenizer) {
yield startToken;
}
// Populate the first character of target sequence with the start character.
let targetSeq = tf.oneHot([startTokenValue], comment_vocab_size).expandDims();
function convertToTensors(featuresData, labelData, testSplit) {
if (featuresData.length !== labelData.length) {
throw new Error('features set and labels set have different numbers of examples');
}
const [shuffledFeatures, shuffledLabels] = shuffleData(featuresData, labelData);
const featuresTensor = tf.tensor2d(shuffledFeatures, [numSamplesPerGesture, totalNumDataPerFile]);
// Create a 1D `tf.Tensor` to hold the labels, and convert the number label
// from the set {0, 1, 2} into one-hot encoding (.e.g., 0 --> [1, 0, 0]).
const labelsTensor = tf.oneHot(tf.tensor1d(shuffledLabels).toInt(), numClasses);
return split(featuresTensor, labelsTensor, testSplit);
}
yield nextWord;
} else if (nextWord == '' || nextWord === '') {
yield '\n';
firstInLine = true;
} else if (nextWord[0] === '>') {
const space = firstInLine || prevWord[0] === '>' ? '' : ' ';
yield space + nextWord[1];
firstInLine = false;
} else {
const space = firstInLine ? '' : ' ';
yield space + nextWord;
firstInLine = false;
}
prevWord = nextWord;
targetSeq = tf.oneHot([sampled_token_index], comment_vocab_size).expandDims();
statesValues = [h, c];
}
}