Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
evaluate (optional): Indicates whether the examples are for evaluation or for training.
no_cache (optional): Force feature conversion and prevent caching. I.e. Ignore cached features even if present.
"""
process_count = self.args["process_count"]
tokenizer = self.tokenizer
output_mode = "classification"
args = self.args
mode = "dev" if evaluate else "train"
if not to_predict:
if isinstance(data, str):
examples = read_examples_from_file(data, mode)
else:
examples = get_examples_from_df(data)
else:
examples = to_predict
no_cache = True
cached_features_file = os.path.join(args["cache_dir"], "cached_{}_{}_{}_{}_{}".format(mode, args["model_type"], args["max_seq_length"], self.num_labels, len(examples)))
if not os.path.isdir(self.args["cache_dir"]):
os.mkdir(self.args["cache_dir"])
if os.path.exists(cached_features_file) and not args["reprocess_input_data"] and not no_cache:
features = torch.load(cached_features_file)
print(f"Features loaded from cache at {cached_features_file}")
else:
print(f"Converting to features started.")