Reading part of a table#
Every other notebook reads whole columns. That is the right default, and on a table of a few thousand rows it is also the fastest thing to do. But a column of a few million rows is a different proposition: reading all of it to look at fifty rows costs the whole column in time and in memory.
This notebook is about asking for less. h5col columns support subscript —
column[17:98], column[-1], column[mask] — and only the rows asked for are
read from the file. We will build a table big enough for the difference to be
visible, then measure it.
import tempfile
import time
from pathlib import Path
import h5py
import numpy as np
import h5col
from h5col import ColumnSpec, LeafValuesSpec, ListColumnSpec, Table, field
print(f"h5col {h5col.__version__} | h5py {h5py.__version__}")
h5col 0.5.0 | h5py 3.16.0
A table worth slicing#
200,000 rows: a station identifier, a temperature that is sometimes missing, and a list column holding a handful of raw samples per row. The chunk size is set small enough that a range read touches only a few chunks.
path = Path(tempfile.gettempdir()) / "readings.h5"
N = 200_000
rng = np.random.default_rng(0)
with h5py.File(path, "w") as f:
table = Table.create(
f.create_group("readings"),
[
ColumnSpec(name="station", dtype=h5col.FixedString(6), chunks=8192),
ColumnSpec(
name="t_air",
dtype="f8",
fill_value=np.nan,
units="degC",
chunks=8192,
),
ListColumnSpec(
name="samples",
values=LeafValuesSpec(dtype="f8"),
nullable=True,
),
],
)
table.append(
{
"station": [f"S{i % 900:05d}" for i in range(N)],
# Every eleventh reading did not arrive.
"t_air": [
None if i % 11 == 0 else float(15 + (i % 200) / 10) for i in range(N)
],
"samples": [
None if i % 97 == 0 else [float(i), float(i + 1), float(i + 2)]
for i in range(N)
],
}
)
print(f"{table.nrows:,} rows written")
print(f"{path.stat().st_size / 1e6:.1f} MB on disk")
200,000 rows written
19.7 MB on disk
Subscript#
A column reads by subscript, and the key can be whatever describes the rows you want. An integer gives that row’s value on its own; anything else gives an array.
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["t_air"]
print("col[17:22] ", col[17:22])
print("col[0] ", repr(col[0]), " <- row 0 is missing")
print("col[1] ", repr(col[1]))
print("col[-1] ", repr(col[-1]), " <- counts back from the end")
print("col[[5, 1, 5]] ", col[[5, 1, 5]], " <- any order, repeats allowed")
print("len(col) ", f"{len(col):,}")
col[17:22] [16.7 16.8 16.9 17.0 17.1]
col[0] masked <- row 0 is missing
col[1] np.float64(15.1)
col[-1] np.float64(34.9) <- counts back from the end
col[[5, 1, 5]] [15.5 15.1 15.5] <- any order, repeats allowed
len(col) 200,000
Missing rows arrive masked, exactly as they do from read(). A single missing
row is numpy.ma.masked, which you can test with is:
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["t_air"]
print("col[0] is np.ma.masked:", col[0] is np.ma.masked)
print("col[1] is np.ma.masked:", col[1] is np.ma.masked)
# A boolean array with one entry per row selects the rows it marks.
missing = col.is_missing()
print(f"\n{missing.sum():,} missing rows of {len(col):,}")
print("first few:", col[missing][:4])
col[0] is np.ma.masked: True
col[1] is np.ma.masked: False
18,182 missing rows of 200,000
first few: [-- -- -- --]
Subscript has nowhere to put a keyword, so it always decodes and always masks.
When you want the raw fill values instead, call read_rows, which takes the
same keys plus masked=:
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["t_air"]
print("masked ", col.read_rows(slice(0, 4)))
print("unmasked ", col.read_rows(slice(0, 4), masked=False))
masked [-- 15.1 15.2 15.3]
unmasked [ nan 15.1 15.2 15.3]
What it saves#
The point of all this is that the file is not read in full. We can count that directly by watching every read h5py performs.
def count_reads(fn):
"""Total elements *fn* pulls out of HDF5, counted at the h5py level."""
real = h5py.Dataset.__getitem__
total = 0
def logged(self, key):
nonlocal total
block = real(self, key)
total += int(np.size(block))
return block
h5py.Dataset.__getitem__ = logged
try:
fn()
finally:
h5py.Dataset.__getitem__ = real
return total
def timed(fn, repeat=3):
return (
min(
(lambda s=time.perf_counter(): (fn(), time.perf_counter() - s)[1])()
for _ in range(repeat)
)
* 1000
)
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["t_air"]
cases = [
("col.read() (whole column)", lambda: col.read()),
("col[100:150]", lambda: col[100:150]),
("col[100]", lambda: col[100]),
("col[[20, 22, 21]]", lambda: col[[20, 22, 21]]),
]
print(f"{'':30} {'elements read':>14} {'ms':>8}")
for label, fn in cases:
print(f"{label:30} {count_reads(fn):>14,} {timed(fn):>8.2f}")
elements read ms
col.read() (whole column) 200,000 0.15
col[100:150] 50 0.05
col[100] 1 0.07
col[[20, 22, 21]] 3 0.07
Fifty rows cost fifty values instead of two hundred thousand. The wall-clock saving here is modest — this column is only 1.6 MB, so reading all of it was never slow — but the elements read are what decides whether a column that does not fit in memory can be worked with at all.
Scattered positions are fetched with coalesced, chunk-aligned reads: h5col
works out which chunks hold the wanted rows and reads those, so two rows at
opposite ends of the column cost two chunks rather than everything between
them.
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["t_air"]
chunk = col.dataset.chunks[0]
print(f"chunk length {chunk:,} rows")
print(
"clustered [20, 22, 21] ",
f"{count_reads(lambda: col[[20, 22, 21]]):>10,} elements",
)
print(
"spanning [0, N-1] ",
f"{count_reads(lambda: col[[0, N - 1]]):>10,} elements",
" <- the two chunks those rows land in, not the span",
)
print("whole column ", f"{count_reads(lambda: col.read()):>10,} elements")
chunk length 8,192 rows
clustered [20, 22, 21] 3 elements
spanning [0, N-1] 11,584 elements <- the two chunks those rows land in, not the span
whole column 200,000 elements
List columns#
List columns take the same keys, and this is where asking for less pays best. Reading a list column into Python has to build an object for every row, which costs far more than the reading does — so cutting the rows cuts the work twice over.
They reach their rows differently, too. A list column finds its values through
an OFFSETS array, so a range read looks up where that range’s values start
and end and reads only that span, at every level of nesting.
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["samples"]
print("col[0] ", col[0], " <- a null row, not an empty one")
print("col[1] ", col[1])
print("col[1:3] ", col[1:3])
print("col[-1] ", col[-1])
col[0] None <- a null row, not an empty one
col[1] [np.float64(1.0), np.float64(2.0), np.float64(3.0)]
col[1:3] [[np.float64(1.0), np.float64(2.0), np.float64(3.0)], [np.float64(2.0), np.float64(3.0), np.float64(4.0)]]
col[-1] [np.float64(199999.0), np.float64(200000.0), np.float64(200001.0)]
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["samples"]
print(f"{'':28} {'elements read':>14} {'ms':>8}")
for label, fn in [
("col.read() (whole column)", lambda: col.read()),
("col[100:150]", lambda: col[100:150]),
("col[100]", lambda: col[100]),
]:
print(f"{label:28} {count_reads(fn):>14,} {timed(fn):>8.2f}")
elements read ms
col.read() (whole column) 993,815 78.12
col[100:150] 251 1.33
col[100] 6 1.28
Unlike a scalar column, there is no chunk coalescing here: scattered positions are served from the range that spans them, from the lowest wanted row to the highest. Rows that sit near each other are almost free; rows at opposite ends of the column cost what the whole column costs, because the span between them is the column. It is never worse than reading it whole.
with h5py.File(path, "r") as f:
col = Table.open(f["readings"])["samples"]
print(
"clustered [20, 22, 21] ",
f"{count_reads(lambda: col[[20, 22, 21]]):>10,} elements",
)
print(
"spanning [0, N-1] ",
f"{count_reads(lambda: col[[0, N - 1]]):>10,} elements",
)
clustered [20, 22, 21] 16 elements
spanning [0, N-1] 993,815 elements
It composes with queries#
A query already reads only the rows it matched. Now the list columns in that result are narrowed too, rather than being read whole and then subset.
with h5py.File(path, "r") as f:
table = Table.open(f["readings"])
selection = table.select(field("t_air") > 34.8)
print(f"{selection.count:,} matching rows")
result = selection.read(["t_air", "samples"])
print("t_air ", result["t_air"][:4])
print("samples", result["samples"][:2])
909 matching rows
t_air [34.9 34.9 34.9 34.9]
samples [[np.float64(199.0), np.float64(200.0), np.float64(201.0)], [np.float64(399.0), np.float64(400.0), np.float64(401.0)]]
One thing to watch#
column[...] and column.dataset[...] are a letter apart and are not the same
read. The second goes straight to h5py: it skips decoding, ignores missing
values, and can hand back rows the table does not consider part of itself.
truncate lowers the row count but leaves the rows above it in place as
reserved storage, which makes the difference easy to see:
trunc_path = Path(tempfile.gettempdir()) / "truncated.h5"
with h5py.File(trunc_path, "w") as f:
t = Table.create(
f.create_group("t"),
[ColumnSpec(name="v", dtype="i8", fill_value=-1)],
)
t.append({"v": np.arange(1000)})
t.truncate(10)
col = t["v"]
print("NROWS ", t.nrows)
print("len(col) ", len(col))
print("col[:].shape ", col[:].shape)
print("col.dataset[:].shape ", col.dataset[:].shape, " <- reserved rows included")
NROWS 10
len(col) 10
col[:].shape (10,)
col.dataset[:].shape (1000,) <- reserved rows included
Use .dataset when you want the stored bytes and nothing else. For reading a
table, column[...] is the one that respects what the table says it contains.
Where to go next#
The Reading into Python chapter covers the same ground in prose, including what each column type hands back.
The Exporting to Arrow notebook shows the other way to read a large list column: it reads the whole thing, but hands the stored buffers to Arrow instead of building a Python list per row, which suits wanting most of a column rather than a slice of it.
The queries section covers selecting rows by predicate rather than by position.