Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
import hub
import numpy as np
def download():
vol = hub.load(name='imagenet/image:val')[400:600]
a = (vol.mean(axis=(1,2,3)) == 0).sum()
print(vol.mean(axis=(1,2,3)) == 0)
mnist = hub.array((50000, 28, 28, 1), name="jason/mnist:v2", dtype='float32')
mnist[0, :] = np.random.random((1, 28, 28, 1)).astype('float32')
print(mnist[0,0,0,0])
# TODO load
mnist = hub.load(name='jason/mnist:v1')
print(mnist[0].shape)
print(mnist[0,0,0,0])
def test_aws_wo_hub_creds():
os.system('mv ~/.hub ~/.hub_arxiv')
import hub
x = hub.array((100, 100, 100), 'image/test:smth', dtype='uint8')
print(x.shape)
os.system('mv ~/.hub_arxiv ~/.hub')
def test_wo_aws_or_hub_creds():
os.system('mv ~/.aws ~/.aws_arxiv')
os.system('mv ~/.hub ~/.hub_arxiv')
try:
import hub
x = hub.array((100, 100, 100), 'image/test:smth', dtype='uint8')
print(x.shape)
except Exception as err:
print('pass', err)
pass
os.system('mv ~/.hub_arxiv ~/.hub')
os.system('mv ~/.aws_arxiv ~/.aws')
import hub
import numpy as np
x = hub.array((10,10,10,10), name="davit/example:1", dtype='uint8')
#[0] = np.zeros((1,10,10,10), dtype='uint8') # need to assign
x[1,0,0,0] = 1
import hub
import numpy as np
shape = (10, 10, 10)
x = hub.array(shape, name="test/example:1", dtype='uint8')
x[10]
with open("mnist.pkl", 'rb') as f:
mnist = pickle.load(f)
return mnist
if __name__ == '__main__':
init()
arrays = load()
chunk_length = 128
for key in arrays:
t1 = time.time()
obj = arrays[key]
shape = obj.shape
chunk_size = np.array(shape)
chunk_size[0] = chunk_length
x = hub.array(shape, name='mnist/mnist_test:{}'.format(key),
chunk_size=chunk_size.tolist(), dtype='uint8')
x[:] = obj
t2 = time.time()
print('uploaded {} {} in {}s'.format(key, shape, t2-t1))
def upload_all():
path = '/home/edward/waymo/training/'
filenames = os.listdir(path)
filenames.sort()
pool = ProcessPool(16)
data = pool.map(frames_tfrecord, map(lambda f: path + f, filenames))
frames = sum(data, 0)
print('Frames in files: {}, Total: {}'.format(data, frames))
start_frame = []
for i in range(0, frames):
start_frame.append(sum(data[:i],0))
dataset_type = 'training'
version = 'v2'
storage = S3(bucket='waymo-dataset-upload')
labels_arr = hub.array(shape=(frames, 2, 6, 30, 7), chunk_size=(100, 2, 6, 30, 7), storage=storage, name='edward/{}-labels:{}'.format(dataset_type, version), backend='s3', dtype='float64')
images_arr = hub.array(compression='jpeg', shape=(frames, 6, 1280, 1920, 3), storage=storage, name='edward/{}-camera-images:{}'.format(dataset_type, version), backend='s3', dtype='uint8', chunk_size=(1, 6, 1280, 1920, 3))
def upload_record(i):
upload_tfrecord(dataset_type, path + filenames[i], version, start_frame[i])
for i in range(0, 5):
print("Stage {}".format(i))
pool.map(upload_record, range(i, len(filenames), 5))
path = '/home/edward/waymo/training/'
filenames = os.listdir(path)
filenames.sort()
pool = ProcessPool(16)
data = pool.map(frames_tfrecord, map(lambda f: path + f, filenames))
frames = sum(data, 0)
print('Frames in files: {}, Total: {}'.format(data, frames))
start_frame = []
for i in range(0, frames):
start_frame.append(sum(data[:i],0))
dataset_type = 'training'
version = 'v2'
storage = S3(bucket='waymo-dataset-upload')
labels_arr = hub.array(shape=(frames, 2, 6, 30, 7), chunk_size=(100, 2, 6, 30, 7), storage=storage, name='edward/{}-labels:{}'.format(dataset_type, version), backend='s3', dtype='float64')
images_arr = hub.array(compression='jpeg', shape=(frames, 6, 1280, 1920, 3), storage=storage, name='edward/{}-camera-images:{}'.format(dataset_type, version), backend='s3', dtype='uint8', chunk_size=(1, 6, 1280, 1920, 3))
def upload_record(i):
upload_tfrecord(dataset_type, path + filenames[i], version, start_frame[i])
for i in range(0, 5):
print("Stage {}".format(i))
pool.map(upload_record, range(i, len(filenames), 5))