Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Prepare the data
data = prepare_from_json(data, self.cntx_left, self.cntx_right, self.tokenizer, lowercase=lowercase, tui_filter=tui_filter)
if category_name is not None:
self.category_name = category_name
# Check is the name there
if self.category_name not in data:
raise Exception("The category name does not exist in this json file.")
data = data[self.category_name]
if not fine_tune:
# Encode the category values
data, self.category_values = encode_category_values(data)
self.i_category_values = {v: k for k, v in self.category_values.items()}
else:
# We already have everything, just get the data
data, _ = encode_category_values(data, vals=self.category_values)
# Convert data tkns to ids
data = tkns_to_ids(data, self.tokenizer)
if not fine_tune:
if model_name == 'lstm':
from medcat.utils.models import LSTM
nclasses = len(self.category_values)
bid = model_config.get("bid", True)
num_layers = model_config.get("num_layers", 2)
input_size = model_config.get("input_size", 300)
hidden_size = model_config.get("hidden_size", 300)
if category_name is not None:
self.category_name = category_name
# Check is the name there
if self.category_name not in data:
raise Exception("The category name does not exist in this json file.")
data = data[self.category_name]
if not fine_tune:
# Encode the category values
data, self.category_values = encode_category_values(data)
self.i_category_values = {v: k for k, v in self.category_values.items()}
else:
# We already have everything, just get the data
data, _ = encode_category_values(data, vals=self.category_values)
# Convert data tkns to ids
data = tkns_to_ids(data, self.tokenizer)
if not fine_tune:
if model_name == 'lstm':
from medcat.utils.models import LSTM
nclasses = len(self.category_values)
bid = model_config.get("bid", True)
num_layers = model_config.get("num_layers", 2)
input_size = model_config.get("input_size", 300)
hidden_size = model_config.get("hidden_size", 300)
dropout = model_config.get("dropout", 0.5)
self.model = LSTM(self.embeddings, self.pad_id, nclasses=nclasses, bid=bid, num_layers=num_layers,
input_size=input_size, hidden_size=hidden_size, dropout=dropout)