Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
data: xarray.DataArray or xarray.Dataset to regrid
lon_name: name of longitude dimension. Set to None if nothing should be done.
lat_name: name of latitude dimension. Set to None if nothing should be done.
OUTPUTS:
data: xarray.DataArray with latitude swapped and
longitude from 0 to 360 degrees.
"""
import xarray as xr
if lat_name is not None:
if data[lat_name][0] > data[lat_name][-1]:
data = data.interp({lat_name:data[lat_name][::-1]},method='nearest')
if lon_name is not None and data[lon_name].min() < 0:
data_low = data.sel({lon_name: slice(0,180)})
data_neg = data.sel({lon_name: slice(-180,-0.001)})
data_neg = data_neg.assign_coords({lon_name: data_neg[lon_name]+360})
data = xr.concat([data_low,data_neg],dim=lon_name)
return data.sortby(lon_name)
else:
return data
def create_pseudo_members(control):
startlist = np.random.randint(c_start, c_end - nlead_years - 1, len(dim_label))
return xr.concat([isel_years(control, start) for start in startlist], dim)
upper_var = ds.targets.sel(**upper_sel) * upper_std + upper_mean
lower_var = ds.targets.sel(**lower_sel) * lower_std + lower_mean
upper_var = upper_var.assign_coords(varlev=[new_var_coord])
lower_var = lower_var.assign_coords(varlev=[new_var_coord])
t_thick = upper_var - lower_var
if ds.attrs['scaling'] == 'True':
t_thick = (t_thick - thick_mean) / thick_std
ds_thick['targets'] = t_thick
# Merge the datasets
print('Concatenating datasets...')
ds_new = xr.concat([ds.sel(**keep_sel), ds_thick], dim='varlev')
# Write to output
print('Writing new file (%s)...' % new_file)
ds_new.to_netcdf(new_file)
else:
band_dim_count += 1
band_dim_index = band_dim_count
band_list.append((band_values, band_dim_index, spectrum_name, spectrum_variables))
# Remove waveband variables from dataset
dropped_var_names = {spectrum_variable.name
for _, _, _, spectrum_variables in band_list
for spectrum_variable in spectrum_variables}
dataset = dataset.drop(dropped_var_names)
# And replace by new vectorized waveband / spectrum variables
for band_values, band_dim_index, spectrum_name, spectrum_variables in band_list:
band_dim_name = 'band' if band_dim_count == 0 else 'band' + str(band_dim_index + 1)
band_coord_var = band_coord_var_factory(band_dim_name, band_values)
spectrum_variable = xr.concat(spectrum_variables, dim=band_dim_name)
time_coord_size = spectrum_variable.sizes.get('time', 0)
if time_coord_size == 1 and spectrum_variable.dims[0] != 'time':
spectrum_variable = spectrum_variable.squeeze('time')
spectrum_variable = spectrum_variable.expand_dims('time')
dataset[spectrum_name] = spectrum_variable
dataset = dataset.assign_coords(**{band_dim_name: band_coord_var})
return dataset
----------
dsets : List[xr.Dataset]
A list of xarray.Dataset(s) to concatenate along an existing dimension.
options : Dict, optional
Additional keyword arguments passed through to
:py:func:`~xarray.concat()`, by default None
Returns
-------
xr.Dataset
xarray Dataset
"""
options = options or {}
try:
return xr.concat(dsets, **options)
except Exception as exc:
logger.error('Failed to join datasets along existing dimension.')
raise exc
dsets = _override_coords(dsets, time_coord_name)
# get static vars from first dataset
first = dsets[0]
def drop_unnecessary_coords(ds):
"""Drop coordinates that do not correspond with dimensions."""
non_dim_coords = set(ds.coords) - set(ds.dims)
non_dim_coords_drop = [
coord for coord in non_dim_coords if time_coord_name not in ds[coord].dims
]
return ds.drop(non_dim_coords_drop)
rest = [drop_unnecessary_coords(ds) for ds in dsets[1:]]
objs_to_concat = [first] + rest
ds = xr.concat(objs_to_concat, dim=time_coord_name, coords='minimal')
new_history = f"\n{datetime.now()} xarray.concat(, dim='{time_coord_name}', coords='minimal')"
if 'history' in attrs:
attrs['history'] += new_history
else:
attrs['history'] = new_history
ds.attrs = attrs
if restore_non_dim_coords:
return _restore_non_dim_coords(ds)
else:
return ds
def apply_horizon(data, windowsteps, extents):
window_forecasts = []
for windowstep in windowsteps:
window_forecast = []
for _, d_from, d_to, resolution in extents:
# Set extents to start from current windowstep
d_from = windowstep + d_from
d_to = windowstep + d_to
if resolution == 'window':
resampled = data.loc[dict(timesteps=slice(d_from, d_to))]
else:
resampled = data.loc[dict(timesteps=slice(d_from, d_to))].resample(timesteps=resolution).mean('timesteps')
window_forecast.append(resampled)
window_forecasts.append(xr.concat(window_forecast, dim='timesteps').drop('timesteps'))
forecast_data = xr.concat(window_forecasts, dim='windowsteps')
forecast_data['windowsteps'] = windowsteps
forecast_data = forecast_data.rename({'timesteps': 'horizonsteps'})
return forecast_data
def __iter__(self):
for ds_batch in self._iterate_batch_dims(self.ds):
if self.preload_batch:
ds_batch.load()
input_generator = self._iterate_input_dims(ds_batch)
if self.concat_input_dims:
new_dim_suffix = '_input'
all_dsets = [_drop_input_dims(ds_input, list(self.input_dims),
suffix=new_dim_suffix)
for ds_input in input_generator]
dsc = xr.concat(all_dsets, dim='input_batch')
new_input_dims = [dim + new_dim_suffix for dim in self.input_dims]
yield _maybe_stack_batch_dims(dsc, new_input_dims)
else:
for ds_input in input_generator:
yield _maybe_stack_batch_dims(ds_input, list(self.input_dims))
dims.remove(self.sampleTag)
except ValueError:
pass #not there, so didn't need to remove
dimsMeta[name] = ','.join(dims)
# store sample tag, IO information, coordinates
self.addMeta('DataSet',{'dims':dimsMeta})
self.addMeta('DataSet',{'general':{'sampleTag':self.sampleTag,
'inputs':','.join(self._inputs),
'outputs':','.join(self._outputs),
'pointwise_meta':','.join(sorted(self._metavars)),
}})
elif action == 'extend':
# TODO compatability check!
# TODO Metadata update?
# merge can change dtypes b/c no NaN int type: self._data.merge(new,inplace=True)
self._data = xr.concat([self._data,new],dim=self.sampleTag)
else:
self.raiseAnError(RuntimeError,'action "{}" was not an expected value for converting array list to dataset!'
.format(action))
# regardless if "replace" or "return", set up scaling factors
self._setScalingFactors()
return new
cnts = []
schk = StatusCheck(ds.station_id.size, stn_chk)
for i in np.arange(ds.station_id.size, step=stn_chk):
da = ds[elem][:,i:(i+stn_chk)].load().loc[start_date:end_date,:]
cnt = da.groupby('time.month').count(dim='time')
cnts.append(cnt)
schk.increment(stn_chk)
cnts = xr.concat(cnts, dim='station_id')
ds.close()
del ds
ds = Dataset(ds_path,'r+')
vname = "obs_cnt_%s_%d_%d"%(elem,ymdL(start_date),ymdL(end_date))
if "mth" not in ds.dimensions.keys():
ds.createDimension('mth', 12)
vmth = ds.createVariable('mth', np.int, ('mth',),
fill_value=False)
vmth[:] = np.arange(1,13)
if vname not in ds.variables.keys():
vcnt = ds.createVariable(vname, np.int, ('mth','station_id'),