Exercise: Basic model skill#
import modelskill as ms
You want do to a simple comparison between model and observation using the fmskill.compare method, but the following code snippet doesn’t work.
Change the code below, so that it works as intended. Hint: look at the documentation
help(modelskill.match)
fn_mod = '../data/SW/ts_storm_4.dfs0'
fn_obs = '../data/SW/eur_Hm0.dfs0'
c = ms.match(fn_obs, fn_mod)
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/matching.py:512, in _parse_single_model(mod, item, gtype)
511 try:
--> 512 return model_result(mod, item=item, gtype=gtype)
513 except ValueError as e:
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/model/factory.py:58, in model_result(data, aux_items, gtype, **kwargs)
56 geometry = GeometryType.from_string(gtype)
---> 58 return _modelresult_lookup[geometry](
59 data=data,
60 aux_items=aux_items,
61 **kwargs,
62 )
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/model/point.py:54, in PointModelResult.__init__(self, data, name, x, y, z, item, quantity, aux_items)
53 if not self._is_input_validated(data):
---> 54 data = _parse_point_input(
55 data,
56 name=name,
57 item=item,
58 quantity=quantity,
59 aux_items=aux_items,
60 x=x,
61 y=y,
62 z=z,
63 )
65 assert isinstance(data, xr.Dataset)
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/timeseries/_point.py:102, in _parse_point_input(data, name, item, quantity, x, y, z, aux_items)
101 valid_items = list(data.data_vars)
--> 102 sel_items = _parse_point_items(valid_items, item=item, aux_items=aux_items)
103 item_name = sel_items.values
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/timeseries/_point.py:37, in _parse_point_items(items, item, aux_items)
36 elif len(items) > 1:
---> 37 raise ValueError(
38 f"Input has more than 1 item, but item was not given! Available items: {items}"
39 )
41 item = _get_name(item, valid_names=items)
ValueError: Input has more than 1 item, but item was not given! Available items: ['Europlatform: Sign. Wave Height', 'K14: Sign. Wave Height', 'F16: Sign. Wave Height', 'F3: Sign. Wave Height', 'Europlatform: Sign. Wave Height, W', 'K14: Sign. Wave Height, W', 'F16: Sign. Wave Height, W', 'F3: Sign. Wave Height, W', 'Europlatform: Sign. Wave Height, S', 'K14: Sign. Wave Height, S', 'F16: Sign. Wave Height, S', 'F3: Sign. Wave Height, S', 'Europlatform: Wind speed', 'K14: Wind speed', 'F16: Wind speed', 'F3: Wind speed', 'Europlatform: Wind direction', 'K14: Wind direction', 'F16: Wind direction', 'F3: Wind direction']
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
Cell In[2], line 4
1 fn_mod = '../data/SW/ts_storm_4.dfs0'
2 fn_obs = '../data/SW/eur_Hm0.dfs0'
----> 4 c = ms.match(fn_obs, fn_mod)
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/matching.py:247, in match(obs, mod, obs_item, mod_item, gtype, max_model_gap, spatial_method)
201 """Match observation and model result data in space and time
202
203 NOTE: In case of multiple model results with different time coverage,
(...)
244 Create a Comparer from observation and model results that are already matched
245 """
246 if isinstance(obs, get_args(ObsInputType)):
--> 247 return _single_obs_compare(
248 obs,
249 mod,
250 obs_item=obs_item,
251 mod_item=mod_item,
252 gtype=gtype,
253 max_model_gap=max_model_gap,
254 spatial_method=spatial_method,
255 )
257 assert isinstance(obs, Collection)
259 if len(obs) > 1 and isinstance(mod, Collection) and len(mod) > 1:
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/matching.py:330, in _single_obs_compare(obs, mod, obs_item, mod_item, gtype, max_model_gap, spatial_method)
327 """Compare a single observation with multiple models"""
328 obs = _parse_single_obs(obs, obs_item, gtype=gtype)
--> 330 mods = _parse_models(mod, mod_item, gtype=gtype)
332 raw_mod_data = {m.name: m.extract(obs, spatial_method) for m in mods}
333 matched_data = match_space_time(
334 observation=obs, raw_mod_data=raw_mod_data, max_model_gap=max_model_gap
335 )
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/matching.py:485, in _parse_models(mod, item, gtype)
483 """Return a list of ModelResult objects"""
484 if isinstance(mod, get_args(MRInputType)):
--> 485 return [_parse_single_model(mod, item=item, gtype=gtype)]
486 elif isinstance(mod, Sequence):
487 return [_parse_single_model(m, item=item, gtype=gtype) for m in mod]
File /opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/site-packages/modelskill/matching.py:514, in _parse_single_model(mod, item, gtype)
512 return model_result(mod, item=item, gtype=gtype)
513 except ValueError as e:
--> 514 raise ValueError(
515 f"Could not compare. Unknown model result type {type(mod)}. {str(e)}"
516 )
517 else:
518 if item is not None:
ValueError: Could not compare. Unknown model result type <class 'str'>. Input has more than 1 item, but item was not given! Available items: ['Europlatform: Sign. Wave Height', 'K14: Sign. Wave Height', 'F16: Sign. Wave Height', 'F3: Sign. Wave Height', 'Europlatform: Sign. Wave Height, W', 'K14: Sign. Wave Height, W', 'F16: Sign. Wave Height, W', 'F3: Sign. Wave Height, W', 'Europlatform: Sign. Wave Height, S', 'K14: Sign. Wave Height, S', 'F16: Sign. Wave Height, S', 'F3: Sign. Wave Height, S', 'Europlatform: Wind speed', 'K14: Wind speed', 'F16: Wind speed', 'F3: Wind speed', 'Europlatform: Wind direction', 'K14: Wind direction', 'F16: Wind direction', 'F3: Wind direction']
When you have fixed the above snippet, you can continue to do the skill assessment
# plot a timeseries of the comparison
# * remove the default title
# * set the limits of the y axis to cover the 0-6m interval
Your colleague, who is very skilled at Excel, wants to make a plot like this one:
# Use the .df property on the comparer object to save the obs and model timeseries as an Excel file ("skill.xlsx")
# you might get an error "No module named 'openpyxl'", the solution is to run `pip install openpyxl`
# calculate the default skill metrics using the skill method
# calculate the skill using the mean absolute percentage error and max error, use the metrics argument
# c.skill(metrics=[__,__])
# import the hit_ratio metric from modelskillill.metrics
# and calculate the ratio when the deviation between model and observation is less than 0.5 m
# hint: use the Observation and Model columns of the dataframe from the .df property you used above
# is the hit ratio ~0.95 ? Does it match with your expectation based on the timeseries plot?
# what about a deviation of less than 0.1m? Pretty accurate wave model...
# hit_ratio(c.df.Observation, __, a=__)
# compare the distribution of modelled and observed values, using the .hist method
# change the number of bins to 10