Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
from pandas import concat, Series, merge,to_numeric
from numpy import append, empty, vstack, NaN,array
from gc import collect
#dates = self.cmaq.dates[self.cmaq.indexdates]
grid1 = geometry.GridDefinition(lons=lon, lats=lat)
vals = array([], dtype=cmaqvar.dtype)
date = array([], dtype='O')
site = array([], dtype=df[site_label].dtype)
print ' Interpolating using ' + interp + ' method'
for i, j in enumerate(dates):
con = df.datetime == j
print j
try:
lats = df[con].Latitude.values
lons = df[con].Longitude.values
grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
if interp.lower() == 'nearest':
val = kd_tree.resample_nearest(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, nprocs=2).squeeze()
elif interp.lower() == 'idw':
val = kd_tree.resample_custom(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, neighbours=n, weight_funcs=weight_func,
nprocs=2).squeeze()
elif interp.lower() == 'gauss':
val = kd_tree.resample_gauss(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
sigmas=r / 2., fill_value=NaN, neighbours=n, nprocs=2).squeeze()
vals = append(vals, val)
dd = empty(lons.shape[0], dtype=date.dtype)
dd[:] = j
date = append(date, dd)
site = append(site, df[con].SCS.values)
collect()
def interp_to_improve(self, cmaqvar, df, interp='nearest', r=12000., n=5, weight_func=lambda r: 1 / r ** 2):
import pandas as pd
from numpy import unique
from pyresample import geometry, kd_tree
from numpy import vstack, NaN
grid1 = geometry.GridDefinition(lons=self.cmaq.longitude, lats=self.cmaq.latitude)
dates = self.cmaq.dates[self.cmaq.indexdates]
# only interpolate to sites with latitude and longitude
df.dropna(subset=['Latitude', 'Longitude'], inplace=True)
vals, index = unique(df.Site_Code, return_index=True)
lats = df.Latitude.values[index]
lons = df.Longitude.values[index]
grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
sites = df.Site_Code.values[index]
utc = df.utcoffset.values[index]
vals = pd.Series(dtype=df.Obs.dtype)
date = pd.Series(dtype=df.datetime.dtype)
site = pd.Series(dtype=df.Site_Code.dtype)
utcoffset = pd.Series(dtype=df.utcoffset.dtype)
for i, j in enumerate(self.cmaq.indexdates):
if interp.lower() == 'idw':
val = kd_tree.resample_custom(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, neighbours=n, weight_funcs=weight_func,
nprocs=2).squeeze()
elif interp.lower() == 'gauss':
val = kd_tree.resample_gauss(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
sigmas=r / 2., fill_value=NaN, neighbours=n, nprocs=2).squeeze()
else:
interp = 'nearest'
elif source_lons.size != source_lats.size or \
source_lons.shape != source_lats.shape:
raise ValueError('Mismatch between lons and lats')
# Remove illegal values
valid_input_index = ((source_lons >= -180) & (source_lons <= 180) &
(source_lats <= 90) & (source_lats >= -90))
if reduce_data:
# Reduce dataset
if (isinstance(source_geo_def, geometry.CoordinateDefinition) and
isinstance(target_geo_def, (geometry.GridDefinition,
geometry.AreaDefinition))) or \
(isinstance(source_geo_def, (geometry.GridDefinition,
geometry.AreaDefinition)) and
isinstance(target_geo_def, (geometry.GridDefinition,
geometry.AreaDefinition))):
# Resampling from swath to grid or from grid to grid
lonlat_boundary = target_geo_def.get_boundary_lonlats()
# Combine reduced and legal values
valid_input_index &= \
data_reduce.get_valid_index_from_lonlat_boundaries(
lonlat_boundary[0],
lonlat_boundary[1],
source_lons, source_lats,
radius_of_influence)
if isinstance(valid_input_index, np.ma.core.MaskedArray):
# Make sure valid_input_index is not a masked array
valid_input_index = valid_input_index.filled(False)
def interp_to_crn(self, cmaqvar, df, interp='nearest', r=12000., n=5, weight_func=lambda r: 1 / r ** 2):
from pyresample import geometry, kd_tree
from pandas import concat
from numpy import append, empty, vstack, NaN
dates = self.cmaq.dates[self.cmaq.indexdates]
lat = self.cmaq.latitude
lon = self.cmaq.longitude
grid1 = geometry.GridDefinition(lons=lon, lats=lat)
vals = array([], dtype=cmaqvar.dtype)
date = array([], dtype='O')
site = array([], dtype=df.SCS.dtype)
print ' Interpolating using ' + interp + ' method'
for i, j in enumerate(dates):
con = df.datetime == j
lats = df[con].Latitude.values
lons = df[con].Longitude.values
grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
if interp.lower() == 'nearest':
val = kd_tree.resample_nearest(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, nprocs=2).squeeze()
elif interp.lower() == 'idw':
val = kd_tree.resample_custom(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, neighbours=n, weight_funcs=weight_func,
nprocs=2).squeeze()
def find_nearest_latlon_xarray(arr,lat=37.102400,lon=-76.392900,radius=12e3):
from pyresample import utils,geometry
from numpy import array,vstack
grid1 = geometry.GridDefinition(lons=arr.longitude,lats=arr.latitude)
grid2 = geometry.GridDefinition(lons=vstack([lon]), lats=vstack([lat]))
row,col = utils.generate_nearest_neighbour_linesample_arrays(grid1,grid2,radius)
row = row.flatten()
col = col.flatten()
return arr.sel(COL=col).sel(ROW=row).squeeze()
def __init__(self, lons, lats, nprocs=1):
"""Initialize GridDefinition."""
super(GridDefinition, self).__init__(lons, lats, nprocs)
if lons.shape != lats.shape:
raise ValueError('lon and lat grid must have same shape')
elif lons.ndim != 2:
raise ValueError('2 dimensional lon lat grid expected')
def interp_to_improve(self, cmaqvar, df, interp='nearest', r=12000., n=5, weight_func=lambda r: 1 / r ** 2):
import pandas as pd
from numpy import unique
from pyresample import geometry, kd_tree
from numpy import vstack, NaN
grid1 = geometry.GridDefinition(lons=self.cmaq.longitude, lats=self.cmaq.latitude)
dates = self.cmaq.dates[self.cmaq.indexdates]
# only interpolate to sites with latitude and longitude
df.dropna(subset=['Latitude', 'Longitude'], inplace=True)
vals, index = unique(df.Site_Code, return_index=True)
lats = df.Latitude.values[index]
lons = df.Longitude.values[index]
grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
sites = df.Site_Code.values[index]
utc = df.utcoffset.values[index]
vals = pd.Series(dtype=df.Obs.dtype)
date = pd.Series(dtype=df.datetime.dtype)
site = pd.Series(dtype=df.Site_Code.dtype)
utcoffset = pd.Series(dtype=df.utcoffset.dtype)
for i, j in enumerate(self.cmaq.indexdates):
if interp.lower() == 'idw':
val = kd_tree.resample_custom(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
from pandas import concat
from numpy import append, empty, vstack, NaN
dates = self.cmaq.dates[self.cmaq.indexdates]
lat = self.cmaq.latitude
lon = self.cmaq.longitude
grid1 = geometry.GridDefinition(lons=lon, lats=lat)
vals = array([], dtype=cmaqvar.dtype)
date = array([], dtype='O')
site = array([], dtype=df.SCS.dtype)
print ' Interpolating using ' + interp + ' method'
for i, j in enumerate(dates):
con = df.datetime == j
lats = df[con].Latitude.values
lons = df[con].Longitude.values
grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
if interp.lower() == 'nearest':
val = kd_tree.resample_nearest(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, nprocs=2).squeeze()
elif interp.lower() == 'idw':
val = kd_tree.resample_custom(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
fill_value=NaN, neighbours=n, weight_funcs=weight_func,
nprocs=2).squeeze()
elif interp.lower() == 'gauss':
val = kd_tree.resample_gauss(grid1, cmaqvar[i, :, :].squeeze(), grid2, radius_of_influence=r,
sigmas=r / 2., fill_value=NaN, neighbours=n, nprocs=2).squeeze()
vals = append(vals, val)
dd = empty(lons.shape[0], dtype=date.dtype)
dd[:] = j
date = append(date, dd)
site = append(site, df[con].SCS.values)
if dest_size_deg < src_size_deg * 0.5:
self.valid_index = pr.data_reduce.get_valid_index_from_lonlat_grid(dest_lon,
dest_lat,
src_lon,
src_lat,
radius_of_influence=3000)
src_lon = src_lon[self.valid_index]
src_lat = src_lat[self.valid_index]
# bounding box [can be used to read src data; not used afterwards yet]
idx_row, idx_col = np.where(self.valid_index)
self.valid_box = (np.min(idx_col), np.min(idx_row),
np.max(idx_col), np.max(idx_row))
self.src_def = pr.geometry.SwathDefinition(lons=src_lon, lats=src_lat)
self.dest_def = pr.geometry.GridDefinition(lons=dest_lon, lats=dest_lat)
# geo2radar
else:
# dest_def
dest_lat = readfile.read(self.file, datasetName='latitude')[0]
dest_lon = readfile.read(self.file, datasetName='longitude')[0]
dest_lat, dest_lon, mask = mark_lalo_anomoly(dest_lat, dest_lon)
# src_def
lat0 = float(self.src_metadata['Y_FIRST'])
lon0 = float(self.src_metadata['X_FIRST'])
lat_step = float(self.src_metadata['Y_STEP'])
lon_step = float(self.src_metadata['X_STEP'])
lat_num = int(self.src_metadata['LENGTH'])
lon_num = int(self.src_metadata['WIDTH'])
lat1 = lat0 + lat_step * (lat_num - 1)
def interp_to_obs_new(var,df,lat,lon,radius=12000.):
from numpy import NaN,vstack
from pyresample import geometry,image
from pandas import to_timedelta,DataFrame
#define CMAQ pyresample grid (source)
grid1 = geometry.GridDefinition(lons=lon,lats=lat)
#get unique sites from df
dfn = df.drop_duplicates(subset=['Latitude','Longitude'])
#define site grid (target)
lats = dfn.Latitude.values
lons = dfn.Longitude.values
grid2 = geometry.GridDefinition(lons=vstack(lons), lats=vstack(lats))
#Create image container
i = image.ImageContainerNearest(var.transpose('ROW','COL','TSTEP').values,grid1,radius_of_influence=radius,fill_value=NaN)
#resample
ii = i.resample(grid2).image_data.squeeze()
#recombine data
e = DataFrame(ii,index=dfn.SCS,columns=var.TSTEP.values)
w = e.stack().reset_index().rename(columns={'level_1':'datetime',0:'CMAQ'})
w = w.merge(dfn.drop(['datetime','datetime_local','Obs'],axis=1),on='SCS',how='left')
w = w.merge(df[['datetime','SCS','Obs']],on=['SCS','datetime'],how='left')
#calculate datetime local
w['datetime_local'] = w.datetime + to_timedelta(w.utcoffset,'H')
return w