Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def get_tiles(self):
acq_min, acq_max, criteria = build_season_date_criteria(self.acq_min, self.acq_max, self.season,
seasons=SEASONS, extend=True)
_log.info("\tcriteria is %s", criteria)
dataset_types = [self.dataset_type]
if self.mask_pqa_apply:
dataset_types.append(DatasetType.PQ25)
tiles = list_tiles_as_list(x=[self.x], y=[self.y], satellites=self.satellites,
acq_min=acq_min, acq_max=acq_max,
dataset_types=dataset_types, include=criteria)
return tiles
def setup_arguments(self):
# Call method on super class
# super(self.__class__, self).setup_arguments()
CellTool.setup_arguments(self)
self.parser.add_argument("--dataset-type", help="The type(s) of dataset to retrieve",
action="store",
dest="dataset_type",
type=dataset_type_arg,
choices=self.get_supported_dataset_types(), default=DatasetType.ARG25, required=True,
metavar=" ".join([s.name for s in self.get_supported_dataset_types()]))
group = self.parser.add_mutually_exclusive_group()
group.add_argument("--band", help="The band(s) to retrieve", action="store", dest="bands", type=str, nargs="+")
group.add_argument("--bands-all", help="Retrieve all bands with NULL values where the band is N/A",
action="store_const", dest="bands", const=BandListType.ALL)
group.add_argument("--bands-common", help="Retrieve only bands in common across all satellites",
action="store_const", dest="bands", const=BandListType.COMMON)
self.parser.set_defaults(bands=BandListType.ALL)
self.parser.add_argument("--output-directory", help="Output directory", action="store", dest="output_directory",
type=writeable_dir, required=True)
def run(self):
stack = list()
for tile in self.get_tiles():
# The Tassel Cap dataset is a virtual dataset derived from the NBAR so it's path is actually the NBAR path
filename = tile.datasets[DatasetType.TCI].path
filename = map_filename_nbar_to_wetness(filename)
filename = os.path.join(self.output_directory, filename)
print "+++", filename
log_mem("Before get data")
data = read_dataset_data(filename, bands=[TciBands.WETNESS],
x=self.x_offset, y=self.y_offset,
x_size=self.chunk_size_x, y_size=self.chunk_size_y)
log_mem("After get data")
# stack.append(data[TciBands.WETNESS])
def requires(self):
_log.debug("SummaryTask.requires()")
from datacube.config import Config
config = Config(os.path.expanduser("~/.datacube/config"))
_log.debug(config.to_str())
x_list = range(self.x_min, self.x_max + 1)
y_list = range(self.y_min, self.y_max + 1)
return [self.create_cell_task(x=cell.x, y=cell.y) for cell in
list_cells(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max,
satellites=[satellite for satellite in self.satellites],
datasets=[DatasetType.ARG25, DatasetType.PQ25, DatasetType.FC25],
database=config.get_db_database(),
user=config.get_db_username(),
password=config.get_db_password(),
host=config.get_db_host(), port=config.get_db_port())]
if DatasetType.DSM in dataset_types:
sql += """
left outer join
(
select
dataset.acquisition_id, tile.dataset_id, tile.x_index, tile.y_index, tile.tile_pathname, tile.tile_type_id, tile.tile_class_id
from tile
join dataset on dataset.dataset_id=tile.dataset_id
where dataset.level_id = %(level_dsm)s
) as dsm on
dsm.x_index=nbar.x_index and dsm.y_index=nbar.y_index
and dsm.tile_type_id=nbar.tile_type_id and dsm.tile_class_id=nbar.tile_class_id
"""
if DatasetType.DEM in dataset_types:
sql += """
left outer join
(
select
dataset.acquisition_id, tile.dataset_id, tile.x_index, tile.y_index, tile.tile_pathname, tile.tile_type_id, tile.tile_class_id
from tile
join dataset on dataset.dataset_id=tile.dataset_id
where dataset.level_id = %(level_dem)s
) as dem on
dem.x_index=nbar.x_index and dem.y_index=nbar.y_index
and dem.tile_type_id=nbar.tile_type_id and dem.tile_class_id=nbar.tile_class_id
"""
if DatasetType.DEM_HYDROLOGICALLY_ENFORCED in dataset_types:
sql += """
left outer join
def output(self):
nbar = self.tile.datasets[DatasetType.ARG25]
filename = os.path.basename(nbar.path)
filename = filename.replace("NBAR", "WETNESS")
filename = filename.replace(".vrt", ".tif")
filename = os.path.join(self.output_directory, filename)
return luigi.LocalTarget(filename)
PQ_MASK_CLOUD_SHADOW_FMASK]
observation_count = empty_array(shape=shape, dtype=numpy.int16, ndv=0)
observation_count_clear = dict()
for mask in masks:
observation_count_clear[mask] = empty_array(shape=shape, dtype=numpy.int16, ndv=0)
metadata = None
for tile in self.get_tiles():
# Get the PQ mask
pq = tile.datasets[DatasetType.PQ25]
data = get_dataset_data(pq, [Pq25Bands.PQ])[Pq25Bands.PQ]
#
# Count any pixels that are no NDV - don't think we should actually have any but anyway
#
# Mask out any no data pixels - should actually be none but anyway
pq = numpy.ma.masked_equal(data, NDV)
# Count the data pixels - i.e. pixels that were NOT masked out
observation_count += numpy.where(data.mask, 0, 1)
#
# Count and pixels that are not masked due to pixel quality
#
data_type = get_dataset_type_datatype(self.dataset_type)
ndv = get_dataset_type_ndv(self.dataset_type)
metadata = None
driver = None
raster = None
acq_min, acq_max, criteria = build_season_date_criteria(self.acq_min, self.acq_max, self.season,
seasons=SEASONS, extend=True)
_log.info("\tacq %s to %s criteria is %s", acq_min, acq_max, criteria)
dataset_types = [self.dataset_type]
if self.mask_pqa_apply:
dataset_types.append(DatasetType.PQ25)
tiles = list_tiles_as_list(x=[self.x], y=[self.y], satellites=self.satellites,
acq_min=acq_min, acq_max=acq_max,
dataset_types=dataset_types, include=criteria)
for index, tile in enumerate(tiles, start=1):
dataset = tile.datasets[self.dataset_type]
assert dataset
# band = dataset.bands[self.band]
# assert band
band = self.band
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None
dataset = self.tile.datasets[self.dataset_type]
metadata = get_dataset_metadata(dataset)
mask = None
# If doing PQA masking then get PQA mask
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask)
# If doing WOFS masking then get WOFS mask
if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets:
mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask)
# TODO - no data value and data type
ndv = get_dataset_ndv(dataset)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
raster_create(self.output().path, [data[b] for b in dataset.bands],
metadata.transform, metadata.projection, ndv, gdal.GDT_Int16,
dataset_metadata=self.generate_raster_metadata(dataset),
band_ids=[b.name for b in dataset.bands])