How to use the hub.array function in hub

To help you get started, we’ve selected a few hub examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github snarkai / Hub / test / example.py View on Github external
import hub
import numpy as np

def download():
    vol = hub.load(name='imagenet/image:val')[400:600]
    a = (vol.mean(axis=(1,2,3)) == 0).sum()
    print(vol.mean(axis=(1,2,3)) == 0)

mnist = hub.array((50000, 28, 28, 1), name="jason/mnist:v2", dtype='float32')
mnist[0, :] = np.random.random((1, 28, 28, 1)).astype('float32')

print(mnist[0,0,0,0])
# TODO load
mnist = hub.load(name='jason/mnist:v1')

print(mnist[0].shape)
print(mnist[0,0,0,0])
github snarkai / Hub / test / test_init.py View on Github external
def test_aws_wo_hub_creds():
    os.system('mv ~/.hub ~/.hub_arxiv')
    import hub
    x = hub.array((100, 100, 100), 'image/test:smth', dtype='uint8')
    print(x.shape)
    os.system('mv ~/.hub_arxiv ~/.hub')
github snarkai / Hub / test / test_init.py View on Github external
def test_wo_aws_or_hub_creds():
    os.system('mv ~/.aws ~/.aws_arxiv')
    os.system('mv ~/.hub ~/.hub_arxiv')
    try:
        import hub
        x = hub.array((100, 100, 100), 'image/test:smth', dtype='uint8')
        print(x.shape)
    except Exception as err:
        print('pass', err)
        pass
    os.system('mv ~/.hub_arxiv ~/.hub')
    os.system('mv ~/.aws_arxiv ~/.aws')
github snarkai / Hub / test / unit_assign.py View on Github external
import hub
import numpy as np
x = hub.array((10,10,10,10), name="davit/example:1", dtype='uint8')
#[0] = np.zeros((1,10,10,10), dtype='uint8') # need to assign
x[1,0,0,0] = 1
github snarkai / Hub / test / test_exceptions.py View on Github external
import hub
import numpy as np

shape = (10, 10, 10)
x = hub.array(shape, name="test/example:1", dtype='uint8')
x[10]
github snarkai / Hub / examples / upload_mnist.py View on Github external
with open("mnist.pkl", 'rb') as f:
        mnist = pickle.load(f)
    return mnist


if __name__ == '__main__':
    init()
    arrays = load()
    chunk_length = 128
    for key in arrays:
        t1 = time.time()
        obj = arrays[key]
        shape = obj.shape
        chunk_size = np.array(shape)
        chunk_size[0] = chunk_length
        x = hub.array(shape, name='mnist/mnist_test:{}'.format(key),
                      chunk_size=chunk_size.tolist(), dtype='uint8')
        x[:] = obj
        t2 = time.time()
        print('uploaded {} {} in {}s'.format(key, shape, t2-t1))
github snarkai / Hub / waymo_upload / waymo_upload.py View on Github external
def upload_all():
    path = '/home/edward/waymo/training/'
    filenames = os.listdir(path)
    filenames.sort()
    pool = ProcessPool(16)
    data = pool.map(frames_tfrecord, map(lambda f: path + f, filenames))
    frames = sum(data, 0)
    print('Frames in files: {}, Total: {}'.format(data, frames))

    start_frame = []
    for i in range(0, frames):
        start_frame.append(sum(data[:i],0))
    dataset_type = 'training'
    version = 'v2'
    storage = S3(bucket='waymo-dataset-upload')
    labels_arr = hub.array(shape=(frames, 2, 6, 30, 7), chunk_size=(100, 2, 6, 30, 7), storage=storage, name='edward/{}-labels:{}'.format(dataset_type, version), backend='s3', dtype='float64')
    images_arr = hub.array(compression='jpeg', shape=(frames, 6, 1280, 1920, 3), storage=storage, name='edward/{}-camera-images:{}'.format(dataset_type, version), backend='s3', dtype='uint8', chunk_size=(1, 6, 1280, 1920, 3))    

    def upload_record(i):
        upload_tfrecord(dataset_type, path + filenames[i], version, start_frame[i])

    for i in range(0, 5):
        print("Stage {}".format(i))
        pool.map(upload_record, range(i, len(filenames), 5))
github snarkai / Hub / waymo_upload / waymo_upload.py View on Github external
path = '/home/edward/waymo/training/'
    filenames = os.listdir(path)
    filenames.sort()
    pool = ProcessPool(16)
    data = pool.map(frames_tfrecord, map(lambda f: path + f, filenames))
    frames = sum(data, 0)
    print('Frames in files: {}, Total: {}'.format(data, frames))

    start_frame = []
    for i in range(0, frames):
        start_frame.append(sum(data[:i],0))
    dataset_type = 'training'
    version = 'v2'
    storage = S3(bucket='waymo-dataset-upload')
    labels_arr = hub.array(shape=(frames, 2, 6, 30, 7), chunk_size=(100, 2, 6, 30, 7), storage=storage, name='edward/{}-labels:{}'.format(dataset_type, version), backend='s3', dtype='float64')
    images_arr = hub.array(compression='jpeg', shape=(frames, 6, 1280, 1920, 3), storage=storage, name='edward/{}-camera-images:{}'.format(dataset_type, version), backend='s3', dtype='uint8', chunk_size=(1, 6, 1280, 1920, 3))    

    def upload_record(i):
        upload_tfrecord(dataset_type, path + filenames[i], version, start_frame[i])

    for i in range(0, 5):
        print("Stage {}".format(i))
        pool.map(upload_record, range(i, len(filenames), 5))