Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_save_and_load_empty(self):
file_path = os.path.join(_get_temporary_directory(), 'test_with_empty.zip')
df = pd.DataFrame()
processor = LargeDataFrameProcessor(max_byte=int(1e+6))
processor.save(df, file_path)
loaded = processor.load(file_path)
pd.testing.assert_frame_equal(loaded, df, check_like=True)
def test_save_and_load(self):
file_path = os.path.join(_get_temporary_directory(), 'test.zip')
df = pd.DataFrame(dict(data=np.random.uniform(0, 1, size=int(1e+6))))
processor = LargeDataFrameProcessor(max_byte=int(1e+6))
processor.save(df, file_path)
loaded = processor.load(file_path)
pd.testing.assert_frame_equal(loaded, df, check_like=True)
def make_large_data_frame_target(self, relative_file_path: str, use_unique_id: bool = True, max_byte=int(2**26)) -> TargetOnKart:
file_path = os.path.join(self.workspace_directory, relative_file_path)
unique_id = self.make_unique_id() if use_unique_id else None
return gokart.target.make_model_target(file_path=file_path,
temporary_directory=self.local_temporary_directory,
unique_id=unique_id,
save_function=gokart.target.LargeDataFrameProcessor(max_byte=max_byte).save,
load_function=gokart.target.LargeDataFrameProcessor.load)
def make_large_data_frame_target(self, relative_file_path: str, use_unique_id: bool = True, max_byte=int(2**26)) -> TargetOnKart:
file_path = os.path.join(self.workspace_directory, relative_file_path)
unique_id = self.make_unique_id() if use_unique_id else None
return gokart.target.make_model_target(file_path=file_path,
temporary_directory=self.local_temporary_directory,
unique_id=unique_id,
save_function=gokart.target.LargeDataFrameProcessor(max_byte=max_byte).save,
load_function=gokart.target.LargeDataFrameProcessor.load)