Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_BasicSeasonalMetrics():
"""
Test BasicSeasonalMetrics.
"""
df = make_some_data()
data = df[['ref', 'k1']]
with warnings.catch_warnings():
warnings.simplefilter("ignore") # many warnings due to test data
metriccalc = MonthsMetricsAdapter(BasicMetrics(other_name='k1'))
res = metriccalc.calc_metrics(data, gpi_info=(0, 0, 0))
should = dict(ALL_n_obs=np.array([366]), dtype='float32')
assert res['ALL_n_obs'] == should['ALL_n_obs']
assert np.isnan(res['ALL_rho'])
def test_BasicMetrics_calculator_metadata():
"""
Test BasicMetrics with metadata.
"""
df = make_some_data()
data = df[['ref', 'k1']]
metadata_dict_template = {'network': np.array(['None'], dtype='U256')}
with warnings.catch_warnings():
warnings.simplefilter("ignore") # many warnings due to test data
metriccalc = BasicMetrics(other_name='k1', calc_tau=False,
metadata_template=metadata_dict_template)
res = metriccalc.calc_metrics(
data, gpi_info=(0, 0, 0, {'network': 'SOILSCAPE'}))
should = dict(network=np.array(['SOILSCAPE'], dtype='U256'),
n_obs=np.array([366]), RMSD=np.array([0.2], dtype='float32'),
BIAS=np.array([-0.2], dtype='float32'), dtype='float32')
assert res['n_obs'] == should['n_obs']
assert np.isnan(res['rho'])
assert res['RMSD'] == should['RMSD']
assert res['BIAS'] == should['BIAS']
assert res['network'] == should['network']
assert np.isnan(res['R'])
# depends on scipy version changed after v1.2.1
def test_BasicSeasonalMetrics_metadata():
"""
Test BasicSeasonalMetrics with metadata.
"""
df = make_some_data()
data = df[['ref', 'k1']]
metadata_dict_template = {'network': np.array(['None'], dtype='U256')}
with warnings.catch_warnings():
warnings.simplefilter("ignore") # many warnings due to test data
metriccalc = MonthsMetricsAdapter(BasicMetrics(
other_name='k1', metadata_template=metadata_dict_template))
res = metriccalc.calc_metrics(
data, gpi_info=(0, 0, 0, {'network': 'SOILSCAPE'}))
assert res['network'] == np.array(['SOILSCAPE'], dtype='U256')
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)}}
datasets = setup_TestDatasets()
dm = DataManager(datasets, 'DS1', read_ts_names={d: 'read' for d in ['DS1', 'DS2', 'DS3']})
process = Validation(
dm, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics})
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
assert sorted(list(results)) == sorted(list(tst_results))
'masking2': {
'class': mds2,
'columns': ['x'],
'args': [],
'kwargs': {'limit': 1000},
'use_lut': False,
'grids_compatible': True}
}
process = Validation(
datasets, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
masking_datasets=mds)
gpi_info = (1, 1, 1)
ref_df = datasets['DS1']['class'].read(1)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning)
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 0
nptest.assert_allclose(new_ref_df.x.values, np.arange(1000, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning)
results = process.calc(*job)
tst = []
assert sorted(list(results)) == sorted(list(tst))
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)}}
datasets = setup_three_with_two_overlapping()
dm = DataManager(datasets, 'DS1', read_ts_names={d: 'read' for d in ['DS1', 'DS2', 'DS3']})
process = Validation(
dm, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics})
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
assert sorted(list(results)) == sorted(list(tst_results))
'kwargs': {'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True},
}}
read_ts_names = {'ASCAT': 'read', 'ISMN': 'read_ts'}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(datasets, 'ISMN', period, read_ts_names=read_ts_names)
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1', metadata_template=metadata_dict_template).calc_metrics},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R', u'time',
u'idx', u'_row_size']
for key, value in metadata_dict_template.items():
vars_should.append(key)
n_obs_should = [384, 357, 482, 141, 251, 1927, 1887, 1652]
'mask_ssf': True}
}}
# The datasets dictionary contains all the information about the datasets to read. The `class` is the dataset class
# to use which we have already initialized. The `columns` key describes which columns of the dataset interest us for
# validation. This a mandatory field telling the framework which other columns to ignore. In this case the columns
# `soil moisture_flag` and `soil moisture_orig_flag` will be ignored by the ISMN reader. We can also specify
# additional keywords that should be given to the `read_ts` method of the dataset reader. In this case we want the
# ASCAT reader to mask the ASCAT soil moisture using the included frozen and snow probabilities as well as the SSF.
# There are also other keys that can be used here. Please see the documentation for explanations.
# In[13]:
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
basic_metrics = metrics_calculators.BasicMetrics(other_name='k1')
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={(2, 2): basic_metrics.calc_metrics},
period=period)
# During the initialization of the Validation class we can also tell it other things that it needs to know. In this
# case it uses the datasets we have specified earlier. The spatial reference is the `'ISMN'` dataset which is the
# second argument. The 'metrics_calculators' argument looks a little bit strange so let's look at it in more detail.
#
# It is a dictionary with a tuple as the key and a function as the value. The key tuple `(n, k)` has the following
# meaning: `n` datasets are temporally matched together and then given in sets of `k` columns to the metric
'mask_ssf': True}
}}
# The datasets dictionary contains all the information about the datasets to read. The `class` is the dataset class
# to use which we have already initialized. The `columns` key describes which columns of the dataset interest us for
# validation. This a mandatory field telling the framework which other columns to ignore. In this case the columns
# `soil moisture_flag` and `soil moisture_orig_flag` will be ignored by the ISMN reader. We can also specify
# additional keywords that should be given to the `read_ts` method of the dataset reader. In this case we want the
# ASCAT reader to mask the ASCAT soil moisture using the included frozen and snow probabilities as well as the SSF.
# There are also other keys that can be used here. Please see the documentation for explanations.
# In[8]:
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
basic_metrics = metrics_calculators.BasicMetrics(other_name='k1')
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={(2, 2): basic_metrics.calc_metrics},
period=period)
# During the initialization of the Validation class we can also tell it other things that it needs to know. In this
# case it uses the datasets we have specified earlier. The spatial reference is the `'ISMN'` dataset which is the
# second argument. The 'metrics_calculators' argument looks a little bit strange so let's look at it in more detail.
#
# It is a dictionary with a tuple as the key and a function as the value. The key tuple `(n, k)` has the following
# meaning: `n` datasets are temporally matched together and then given in sets of `k` columns to the metric