Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async function main() {
const sentenceEncoder = await UniversalSentenceEncoder.load();
// The code for the encoder gives too many warnings.
tf.disableDeprecationWarnings();
async function getCentroid(intent) {
const filename = `train_${intent}_full.json`;
let data = fs.readFileSync(path.join(dataPath, intent, filename), 'utf8');
data = JSON.parse(data)[intent];
console.log(`Found ${data.length} samples for ${intents[intent]}.`);
if (trainSplit < 1) {
data = data.slice(0, Math.round(trainSplit * data.length));
}
const queries = data.map(datum => datum.data.map(o => o.text).join(""));
console.log(` Training with ${queries.length} samples.`);
const chunkSize = 128;
async function run(srcPath, outPath, batchSize) {
console.log('Start');
const use = await useLoader.load();
console.log('Loaded Universal Sentence Encoder');
const csvData = loadCSV(srcPath);
const queries = csvData.map(q => q.query);
await embedTokens(queries, use, batchSize, outPath);
console.log('Done');
}
async function loadUSE() {
if (use == null) {
use = await useLoader.load();
}
return use;
}
async setTransformInputMethod() {
if (this.state.contractInfo.encoder === 'universal sentence encoder') {
this.setState({ inputType: INPUT_TYPE_TEXT });
UniversalSentenceEncoder.load().then(use => {
this.transformInput = async (query) => {
const embeddings = await use.embed(query);
let embedding = tf.tidy(_ => {
const emb = embeddings.gather(0);
if (this.state.featureIndices !== undefined && this.state.featureIndices.length > 0) {
return emb.gather(this.state.featureIndices).arraySync();
}
return emb.arraySync();
});
embeddings.dispose();
return this.normalize(embedding).then(normalizedEmbedding => {
return normalizedEmbedding.map(v => this.web3.utils.toHex(v));
});
};
this.transformInput = this.transformInput.bind(this);
});
async train() {
const promises = [];
if (!model) promises.push(UniversalSentenceEncoder.load());
else promises.push(Promise.resolve(model));
if (!tokenizer) promises.push(UniversalSentenceEncoder.loadTokenizer());
else promises.push(Promise.resolve(tokenizer));
const USE = await Promise.all(promises);
if (!model) model = USE[ 0 ];
if (!tokenizer) tokenizer = USE[ 1 ];
this.model = model;
this.tokenizer = tokenizer;
return this.model;
}
/**
get_embeddings(list_sentences, callback) {
use.load().then(model => {
model.embed(list_sentences).then(embeddings => {
callback(embeddings);
});
});
}
async function loadUSE() {
if (use == null) {
use = await useLoader.load();
}
return use;
}
async loadTokenizer() {
return use.loadTokenizer();
}
async train() {
const promises = [];
if (!model) promises.push(load());
else promises.push(Promise.resolve(model));
if (!tokenizer) promises.push(loadTokenizer());
else promises.push(Promise.resolve(tokenizer));
const USE = await Promise.all(promises);
if (!model) model = USE[ 0 ];
if (!tokenizer) tokenizer = USE[ 1 ];
this.model = model;
this.tokenizer = tokenizer;
return this.model;
}
/**
async train() {
const promises = [];
if (!model) promises.push(UniversalSentenceEncoder.load());
else promises.push(Promise.resolve(model));
if (!tokenizer) promises.push(UniversalSentenceEncoder.loadTokenizer());
else promises.push(Promise.resolve(tokenizer));
const USE = await Promise.all(promises);
if (!model) model = USE[ 0 ];
if (!tokenizer) tokenizer = USE[ 1 ];
this.model = model;
this.tokenizer = tokenizer;
return this.model;
}
/**