Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if use_chunks:
logger.debug(
f'Computing metadata for file {raster_path} using more than '
f'{RasterDriver._LARGE_RASTER_THRESHOLD // 10**6}M pixels, iterating '
'over chunks'
)
if use_chunks and not has_crick:
warnings.warn(
'Processing a large raster file, but crick failed to import. '
'Reading whole file into memory instead.', exceptions.PerformanceWarning
)
use_chunks = False
if use_chunks:
raster_stats = RasterDriver._compute_image_stats_chunked(src)
else:
raster_stats = RasterDriver._compute_image_stats(src, max_shape)
if raster_stats is None:
raise ValueError(f'Raster file {raster_path} does not contain any valid data')
row_data.update(raster_stats)
row_data['bounds'] = bounds
row_data['metadata'] = extra_metadata
return row_data
f'Computing metadata for file {raster_path} using more than '
f'{RasterDriver._LARGE_RASTER_THRESHOLD // 10**6}M pixels, iterating '
'over chunks'
)
if use_chunks and not has_crick:
warnings.warn(
'Processing a large raster file, but crick failed to import. '
'Reading whole file into memory instead.', exceptions.PerformanceWarning
)
use_chunks = False
if use_chunks:
raster_stats = RasterDriver._compute_image_stats_chunked(src)
else:
raster_stats = RasterDriver._compute_image_stats(src, max_shape)
if raster_stats is None:
raise ValueError(f'Raster file {raster_path} does not contain any valid data')
row_data.update(raster_stats)
row_data['bounds'] = bounds
row_data['metadata'] = extra_metadata
return row_data
exceptions.PerformanceWarning, stacklevel=3
)
with rasterio.open(raster_path) as src:
if src.nodata is None and not cls._has_alpha_band(src):
warnings.warn(
f'Raster file {raster_path} does not have a valid nodata value, '
'and does not contain an alpha band. No data will be masked.'
)
bounds = warp.transform_bounds(
src.crs, 'epsg:4326', *src.bounds, densify_pts=21
)
if use_chunks is None and max_shape is None:
use_chunks = src.width * src.height > RasterDriver._LARGE_RASTER_THRESHOLD
if use_chunks:
logger.debug(
f'Computing metadata for file {raster_path} using more than '
f'{RasterDriver._LARGE_RASTER_THRESHOLD // 10**6}M pixels, iterating '
'over chunks'
)
if use_chunks and not has_crick:
warnings.warn(
'Processing a large raster file, but crick failed to import. '
'Reading whole file into memory instead.', exceptions.PerformanceWarning
)
use_chunks = False
if use_chunks:
args = (
'epsg:4326',
'epsg:4326',
2 * 10**6,
10**6,
-10, -10, 10, 10
)
# GDAL defaults don't round-trip non-square pixels
_, rio_width, rio_height = calculate_default_transform(*args)
assert rio_width != args[2]
assert rio_height != args[3]
# we do!
our_transform, our_width, our_height = RasterDriver._calculate_default_transform(*args)
assert our_width == args[2]
assert our_height == args[3]
def test_bench_compute_metadata(benchmark, big_raster_file_nodata, big_raster_file_mask,
chunks, raster_type):
from terracotta.drivers.raster_base import RasterDriver
if raster_type == 'nodata':
raster_file = big_raster_file_nodata
elif raster_type == 'masked':
raster_file = big_raster_file_mask
benchmark(RasterDriver.compute_metadata, str(raster_file), use_chunks=chunks)
raster_file = big_raster_file_nomask
elif nodata_type == 'nan':
raster_file = raster_file_float
if use_chunks:
pytest.importorskip('crick')
with rasterio.open(str(raster_file)) as src:
data = src.read(1, masked=True)
valid_data = np.ma.masked_invalid(data).compressed()
convex_hull = convex_hull_exact(src)
# compare
if nodata_type == 'none':
with pytest.warns(UserWarning) as record:
mtd = RasterDriver.compute_metadata(str(raster_file), use_chunks=use_chunks)
assert 'does not have a valid nodata value' in str(record[0].message)
else:
mtd = RasterDriver.compute_metadata(str(raster_file), use_chunks=use_chunks)
np.testing.assert_allclose(mtd['valid_percentage'], 100 * valid_data.size / data.size)
np.testing.assert_allclose(mtd['range'], (valid_data.min(), valid_data.max()))
np.testing.assert_allclose(mtd['mean'], valid_data.mean())
np.testing.assert_allclose(mtd['stdev'], valid_data.std())
# allow some error margin since we only compute approximate quantiles
np.testing.assert_allclose(
mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
rtol=2e-2, atol=valid_data.max() / 100
)
def test_compute_metadata_invalid_options(big_raster_file):
from terracotta.drivers.raster_base import RasterDriver
with pytest.raises(ValueError):
RasterDriver.compute_metadata(str(big_raster_file), max_shape=(256, 256), use_chunks=True)
with pytest.raises(ValueError):
RasterDriver.compute_metadata(str(big_raster_file), max_shape=(256, 256, 1))
raster_file = big_raster_file_nomask
elif nodata_type == 'nan':
raster_file = raster_file_float
with rasterio.open(str(raster_file)) as src:
data = src.read(1, masked=True)
valid_data = np.ma.masked_invalid(data).compressed()
convex_hull = convex_hull_exact(src)
# compare
if nodata_type == 'none':
with pytest.warns(UserWarning) as record:
mtd = RasterDriver.compute_metadata(str(raster_file), max_shape=(512, 512))
assert 'does not have a valid nodata value' in str(record[0].message)
else:
mtd = RasterDriver.compute_metadata(str(raster_file), max_shape=(512, 512))
np.testing.assert_allclose(mtd['valid_percentage'], 100 * valid_data.size / data.size, atol=1)
np.testing.assert_allclose(
mtd['range'], (valid_data.min(), valid_data.max()), atol=valid_data.max() / 100
)
np.testing.assert_allclose(mtd['mean'], valid_data.mean(), rtol=0.02)
np.testing.assert_allclose(mtd['stdev'], valid_data.std(), rtol=0.02)
np.testing.assert_allclose(
mtd['percentiles'],
np.percentile(valid_data, np.arange(1, 100)),
atol=valid_data.max() / 100, rtol=0.02
)
assert geometry_mismatch(shape(mtd['convex_hull']), convex_hull) < 0.05
_ERROR_ON_CONNECT = (
'Could not connect to database. Make sure that the given path points '
'to a valid Terracotta database, and that you ran driver.create().'
)
@contextlib.contextmanager
def convert_exceptions(msg: str) -> Iterator:
"""Convert internal sqlite exceptions to our InvalidDatabaseError"""
try:
yield
except sqlite3.OperationalError as exc:
raise exceptions.InvalidDatabaseError(msg) from exc
class SQLiteDriver(RasterDriver):
"""An SQLite-backed raster driver.
Assumes raster data to be present in separate GDAL-readable files on disk or remotely.
Stores metadata and paths to raster files in SQLite.
This is the simplest Terracotta driver, as it requires no additional infrastructure.
The SQLite database is simply a file that can be stored together with the actual
raster files.
Note:
This driver requires the SQLite database to be physically present on the server.
For remote SQLite databases hosted on S3, use
:class:`~terracotta.drivers.sqlite_remote.RemoteSQLiteDriver`.
The SQLite database consists of 4 different tables: