Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_pytorch():
# Create arrays
datahub = hub.fs("./data/cache").connect()
images = datahub.array(
name="test/dataloaders/images3",
shape=(100, 100, 100),
chunk=(1, 100, 100),
dtype="uint8",
)
labels = datahub.array(
name="test/dataloaders/labels3", shape=(100, 1), chunk=(100, 1), dtype="uint8"
)
# Create dataset
ds = datahub.dataset(
name="test/loaders/dataset2", components={"images": images, "labels": labels}
)
# Transform to Pytorch
train_dataset = ds.to_pytorch()
# Create data loader
def test_broadcasting():
print("- Broadcasting")
datahub = hub.fs("./data/cache").connect()
shape = (100, 100, 100)
chunk = (50, 50, 50)
x = datahub.array(name="test/example:3", shape=shape, chunk=chunk, dtype="uint8")
x[0, 0, 0] = 11
assert x[0, 0, 0] == 11
x[0] = 10
assert x[0].mean() == 10
x[1] = np.ones((100, 100), dtype="uint8")
assert x[1].mean() == 1
x[3, 90, :] = np.ones((1, 1, 100), dtype="uint8")
assert x[3, 90].mean() == 1
print("passed")
def test_dataset():
datahub = hub.fs("./data/cache").connect()
x = datahub.array(name='test/example:input', shape = (100, 25, 25), chunk=(20, 5, 5), dtype='uint8')
y = datahub.array(name='test/example:label', shape = (100, 4), chunk = (20, 2), dtype='uint8')
ds = datahub.dataset(components={
'input': x,
'label': y
}, name='test/dataset:train3')
assert ds[0]['input'].shape == (25, 25)
assert ds['input'].shape[0] == 100 # return single array
assert ds['label', 0].mean() == 0 # equivalent ds['train'][0]
def test_to_tensorflow():
print("testing Tensorflow")
conn = hub.fs("./data/cache").connect()
ds = conn.open("test/loaders/dataset2")
# Transform to Tensorflow
train_dataset = ds.to_tensorflow()
batch = next(iter(train_dataset.batch(batch_size=16)))
assert batch["images"].shape == (16, 100, 100)
# TODO create dataloader
def test_dynamic_array():
conn = hub.fs("./data/cache").connect()
arr = conn.array_create(
"test/dynamic_array_3",
shape=(10, 8, 4, 12),
chunk=(5, 4, 2, 6),
dtype="uint8",
dsplit=2,
)
arr.darray[0:10, 0:5] = (2, 12)
arr.darray[0:10, 5:8] = (6, 14)
assert arr[5, 3, :, :].shape == (2, 12)
assert arr[5, 6, :, :].shape == (6, 14)
assert arr[5, 4:6, :, :].shape == (2, 6, 14)
def test_simple_upload_download():
print("- Simple Chunk Upload and Download")
datahub = hub.fs("./data/cache").connect()
shape = (10, 10, 10, 10)
chunk = (5, 5, 5, 5)
datahub = hub.fs("./data/cache").connect()
x = datahub.array(name="test/example:1", shape=shape, chunk=chunk, dtype="uint8")
x[0] = np.ones((1, 10, 10, 10), dtype="uint8")
assert x[0].mean() == 1
print("passed")
def test_open_array():
print("- Loading arrays")
datahub = hub.fs("./data/cache").connect()
x = datahub.open(name="test/example:4")
print(x.shape)
assert np.all(x.shape == np.array((10, 10, 10)))
print("passed")
def test_chunk_shape():
print("- Chunk shape")
datahub = hub.fs("./data/cache").connect()
shape = (10, 10, 10)
chunk = (5, 5, 5)
x = datahub.array(name="test/example:4", shape=shape, chunk=chunk, dtype="uint8")
x[0:5, 0:5, 0:5] = 0
print("passed")
def test_multiple_upload_download():
datahub = hub.fs("./data/cache").connect()
shape = (10, 10, 10, 10)
chunk = (5, 5, 5, 5)
x = datahub.array(name="test/example:1", shape=shape, chunk=chunk, dtype="uint8")
x[0:3] = np.ones((3, 10, 10, 10), dtype="uint8")
assert x[0:3].mean() == 1
print("passed")
import math
import numpy as np
import itertools
import io
# tf.enable_eager_execution()
# from waymo_open_dataset.utils import range_image_utils
# from waymo_open_dataset.utils import transform_utils
# from waymo_open_dataset.utils import frame_utils
# from waymo_open_dataset import dataset_pb2 as open_dataset
import hub
from PIL import Image
# client = hub.gs('snark_waymo_open_dataset', creds_path='.creds/gs.json').connect()
client = hub.fs('/drive/upload').connect()
arr = client.array_open('validation/images')
for i in range(0, 5):
img = arr[10, i]
print(img.shape)
Image.fromarray(img, 'RGB').save(f'output/image-{i}.jpg')