API Summary
Summary of the most important Hub API commands.
Please note that the code examples below may not run standalone.
Creating and Loading Hub Datasets
1
import hub
2
3
# Load a Hub Dataset if it already exists (same as hub.load), or initialize
4
# a new Hub Dataset if it does not already exist (same as hub.empty)
5
ds = hub.dataset('./local_path') # Local path
6
ds = hub.dataset('hub://username/dataset_name') # Activeloop Platform Storage
7
ds = hub.dataset('s3://bucket_name/dataset_name') # AWS S3
8
ds = hub.dataset('gcs://bucket_name/dataset_name') # Google Cloud Storage
9
ds = hub.dataset('mem://dataset_name') # In-memory dataset that is not stored permanently
10
# This is useful for testing.
11
# Load a Hub Dataset
12
ds = hub.load('dataset_path') # Can use any of the paths above
13
14
# Create an empty Hub Dataset
15
ds = hub.empty('dataset_path') # Can use any of the paths above
16
17
# Create an Hub Dataset with the same tensors as another dataset
18
ds = hub.like(ds_object or 'dataset_path')
19
20
# Automatically create a Hub Dataset
21
ds = hub.ingest('source_path', 'hub_dataset_path')
22
ds = hub.ingest_kaggle('kaggle_path', 'hub_dataset_path')
23
24
# Delete a Hub Dataset
25
ds.delete()
Copied!
Creating Tensors and Adding Data
1
# Create a tensor
2
# Specifying htype is recommended for maximizing performance.
3
# Specifying dtype is required if you desire a different dtype compared
4
# to the default dtype for the specified htype.
5
ds.create_tensor('my_tensor', htype = 'bbox', dtype = 'int32')
6
7
# Specifiying the correct compression is critical for images, videos, audio and
8
# other rich data types.
9
ds.create_tensor('songs', htype = 'audio', sample_compression = 'mp3')
10
11
# Create tensor hierarchies
12
ds.create_group('my_group')
13
ds.my_group.create_tensor('my_tensor')
14
ds.create_tensor('my_group/my_tensor') #Automatically creates the group 'my_group'
15
16
# Append a single sample to multiple tensors at once
17
ds.append('tensor_1': np.ones((1,4)), 'tensor_2': hub.read('image.jpg'))
18
ds.my_group.append('tensor_1': np.ones((1,4)), 'tensor_2': hub.read('image.jpg'))
19
20
# For hub.read, if any tensor has the same compression as the file,
21
# the bytes are directly stored in the tensor. Otherwise, the
22
# data is decompressed/recompressed accordingly.
23
24
# Append one-tensor at-a-time
25
ds.my_tensor.append(np.ones((1,4)))
26
ds.my_tensor.append(hub.read('image.jpg'))
27
28
# Append multiple samples at the end of a tensor. The first axis in the
29
# numpy array is assumed to be the sample axis for the tensor
30
ds.my_tensor.extend(np.ones((5,1,4)))
31
32
# Append multiple samples at the end of a tensor.
33
ds.my_tensor.extend([np.ones((1,4)), np.ones((3,4)), np.ones((2,4))])
Copied!
Adding User-Specified Metadata to Datasets and Tensors
1
# Add or update dataset metadata
2
ds.info.update(key1 = 'text', key2 = number)
3
# Also can run
4
# ds.info.update({'key1'='value1', 'key2' = num_value})
5
6
# Add or update tensor metadata
7
ds.my_tensor.info.update(key1 = 'text', key2 = number)
8
9
#Delete metadata
10
ds.info.delete() #Delete all metadata
11
ds.info.delete('key1') #Delete 1 key in metadata
12
ds.info.delete(['key1', 'key2']) #Delete multiple keys in metadata
Copied!
Maximizing performance
1
# Data gets written to long-terms storage at the end of the 'with'
2
# block or whenever the cache is full. This minimizes the number of
3
# write operations during dataset creation.
4
5
ds = hub.load('dataset_path')
6
7
with ds:
8
9
ds.create_tensor('my_tensor')
10
11
for i in range(10):
12
ds.my_tensor.append(i)
Copied!
Accessing Tensor Data
1
# Read tensor sample
2
np_array = ds.my_tensor[0].numpy()
3
text = ds.my_text_tensor[0].data()
4
bytes = ds.my_tensor[0].tobytes()
5
6
# Read tensor samples from tensors in a hierarchical group
7
np_array_1 = ds.my_group.my_tensor_1[0].numpy()
8
np_array_2 = ds.my_group.my_tensor_2[0].numpy()
9
10
# Read multiple tensor samples into numpy array
11
# Returns an error if tensor samples do not have equal shape
12
np_array = ds.my_tensor[0:10].numpy()
13
14
# Read multiple tensor samples into a list of numpy arrays
15
np_array_list = ds.my_tensor[0:10].numpy(aslist=True)
Copied!
Connecting Hub Datasets to ML Frameworks
1
# PyTorch Dataloader
2
dataloader = ds.pytorch(batch_size = 16, num_workers = 2)
3
4
# TensorFlow Dataset
5
ds_tensorflow = ds.tensorflow()
Copied!
Versioning Datasets
1
# Commit data
2
commit_id = ds.commit('Added 100 images of trucks')
3
4
# Print the commit log
5
log = ds.log()
6
7
# Checkout a branch or commit
8
ds.checkout('label_bug_fixes' or commit_id)
9
10
# Create a new branch
11
ds.checkout('new_branch', create = True)
Copied!
Last modified 2d ago