Robust Script for getting variable metadata

This is a fairly robust way of getting the details of the variables. We avoid the possible collisions in catalog merges by loading each file individually as an xarray dataset.

[1]:
# basics
import intake
import numpy as np
import xarray as xr
import dask  # memory-efficient parallel computation and delayed execution (lazy evaluation).

# panadas for the tables
import pandas as pd

pd.set_option("display.max_rows", None)
%run gem_helpers.ipynb

Choose catalog:

[2]:
catalog_file = "/work/ka1081/Catalogs/dyamond-nextgems.json"
my_simulation_id = "dpp0066"
cat = intake.open_esm_datastore(catalog_file).search(simulation_id=my_simulation_id)
cat

ICON-ESM catalog with 7 dataset(s) from 8114 asset(s):

unique
variable_id 155
project 1
institution_id 1
source_id 1
experiment_id 1
simulation_id 1
realm 2
frequency 4
time_reduction 2
grid_label 1
level_type 3
time_min 406
time_max 406
grid_id 1
format 1
uri 8114

loading data:

[3]:
times = get_from_cat(cat, "time_min").values[
    [0, -1]
]  # we assume that all files are present at the first or last timestep. Iterate over more, if you have less trust in the cleanness of the run
all_atts = []
for time in times:
    files = cat.search(time_min=time)
    for uri in get_from_cat(files, "uri")["uri"].values:
        data = xr.open_dataset(uri)
        all_atts.extend(
            dict(variable_id=var, **data[var].attrs) for var in data.data_vars
        )  # we take the detour via dict and ** to add the variable name
frame = pd.DataFrame(all_atts)
frame = frame.drop_duplicates().sort_values(list(frame)).reset_index(drop=True)
frame
[3]:
variable_id standard_name long_name units param CDI_grid_type number_of_grid_in_reference level_type code
0 A_tracer_v_to A_tracer_v_to sea water potential temperature(A_tracer_v) kg/kg NaN unstructured 1.0 NaN 255.0
1 A_veloc_v A_veloc_v vertical velocity diffusion kg/kg NaN unstructured 3.0 NaN 255.0
2 FrshFlux_Evaporation_global FrshFlux_Evaporation FrshFlux_Evaporation_global m/s NaN NaN NaN NaN 255.0
3 FrshFlux_Precipitation_global FrshFlux_Precipitation FrshFlux_Precipitation m/s NaN NaN NaN NaN 255.0
4 FrshFlux_Runoff_global FrshFlux_Runoff FrshFlux_Runoff m/s NaN NaN NaN NaN 255.0
5 FrshFlux_SnowFall_global FrshFlux_SnowFall FrshFlux_SnowFall_global m/s NaN NaN NaN NaN 255.0
6 FrshFlux_TotalIce_global FrshFlux_TotalIce FrshFlux_TotalIce m/s NaN NaN NaN NaN 255.0
7 FrshFlux_TotalOcean_global FrshFlux_TotalOcean FrshFlux_TotalOcean m/s NaN NaN NaN NaN 255.0
8 FrshFlux_VolumeIce_global FrshFlux_VolumeIce FrshFlux_VolumeIce m/s NaN NaN NaN NaN 255.0
9 FrshFlux_VolumeTotal_global FrshFlux_VolumeTotal FrshFlux_VolumeTotal m/s NaN NaN NaN NaN 255.0
10 HeatFlux_Total_global HeatFlux_Total HeatFlux_Total W/m2 NaN NaN NaN NaN 255.0
11 Qbot Qbot Conductive heat flux at ice-ocean interface W/m^2 NaN unstructured 1.0 NaN 255.0
12 Qtop Qtop Energy flux available for surface melting W/m^2 NaN unstructured 1.0 NaN 255.0
13 Wind_Speed_10m Wind_Speed_10m Wind Speed at 10m height m/s NaN unstructured 1.0 NaN 255.0
14 amoc26n amoc26n amoc26n Sv NaN NaN NaN NaN 255.0
15 atlantic_hfbasin atlantic_hfbasin atlantic northward ocean heat transport W NaN NaN NaN NaN 148.0
16 atlantic_hfl atlantic_hfl atlantic implied heat transport W NaN NaN NaN NaN 148.0
17 atlantic_moc atlantic_moc atlantic meridional overturning Sv NaN NaN NaN NaN 148.0
18 atlantic_sltbasin atlantic_sltbasin atlantic northward ocean salt transport kg s-1 NaN NaN NaN NaN 148.0
19 atlantic_wfl atlantic_wfl atlantic implied freshwater transport m3s-1 NaN NaN NaN NaN 148.0
20 atmos_fluxes_FrshFlux_Evaporation atmos_fluxes_FrshFlux_Evaporation atmos_fluxes_FrshFlux_Evaporation [m/s] NaN unstructured 1.0 NaN 255.0
21 atmos_fluxes_FrshFlux_Precipitation atmos_fluxes_FrshFlux_Precipitation atmos_fluxes_FrshFlux_Precipitation [m/s] NaN unstructured 1.0 NaN 255.0
22 atmos_fluxes_FrshFlux_Runoff atmos_fluxes_FrshFlux_Runoff atmos_fluxes_FrshFlux_Runoff [m/s] NaN unstructured 1.0 NaN 255.0
23 atmos_fluxes_FrshFlux_SnowFall atmos_fluxes_FrshFlux_SnowFall atmos_fluxes_FrshFlux_SnowFall [m/s] NaN unstructured 1.0 NaN 255.0
24 atmos_fluxes_HeatFlux_Latent atmos_fluxes_HeatFlux_Latent atmos_fluxes_HeatFlux_Latent [W/m2] NaN unstructured 1.0 NaN 255.0
25 atmos_fluxes_HeatFlux_LongWave atmos_fluxes_HeatFlux_LongWave atmos_fluxes_HeatFlux_LongWave [W/m2] NaN unstructured 1.0 NaN 255.0
26 atmos_fluxes_HeatFlux_Sensible atmos_fluxes_HeatFlux_Sensible atmos_fluxes_HeatFlux_Sensible [W/m2] NaN unstructured 1.0 NaN 255.0
27 atmos_fluxes_HeatFlux_ShortWave atmos_fluxes_HeatFlux_ShortWave atmos_fluxes_HeatFlux_ShortWave [W/m2] NaN unstructured 1.0 NaN 255.0
28 atmos_fluxes_HeatFlux_Total atmos_fluxes_HeatFlux_Total atmos_fluxes_HeatFlux_Total [W/m2] NaN unstructured 1.0 NaN 255.0
29 atmos_fluxes_stress_x atmos_fluxes_stress_x atmos_fluxes_stress_x Pa NaN unstructured 1.0 NaN 255.0
30 atmos_fluxes_stress_xw atmos_fluxes_stress_xw atmos_fluxes_stress_xw Pa NaN unstructured 1.0 NaN 255.0
31 atmos_fluxes_stress_y atmos_fluxes_stress_y atmos_fluxes_stress_y Pa NaN unstructured 1.0 NaN 255.0
32 atmos_fluxes_stress_yw atmos_fluxes_stress_yw atmos_fluxes_stress_yw Pa NaN unstructured 1.0 NaN 255.0
33 cl cl cloud area fraction m2 m-2 22.6.0 unstructured 1.0 NaN NaN
34 cli cli specific cloud ice content kg kg-1 82.1.0 unstructured 1.0 NaN NaN
35 clivi total_cloud_ice vertically integrated cloud ice kg m-2 70.1.0 unstructured 1.0 atmosphere NaN
36 cllvi total_cloud_water vertically integrated cloud water kg m-2 69.1.0 unstructured 1.0 atmosphere NaN
37 clt clt total cloud cover m2 m-2 1.6.0 unstructured 1.0 NaN NaN
38 clw clw specific cloud water content kg kg-1 22.1.0 unstructured 1.0 NaN NaN
39 conc conc ice concentration in each ice class NaN NaN unstructured 1.0 NaN 255.0
40 cptgz cptgz dry static energy m2 s-2 NaN unstructured 1.0 NaN 255.0
41 cptgzvi vertically integrated dry static energy vert_int_dry_static_energy m2 s-2 NaN unstructured 1.0 atmosphere 255.0
42 evspsbl evap evaporation kg m-2 s-1 6.1.0 unstructured 1.0 NaN NaN
43 global_heat_content global_heat_content global_heat_content J NaN NaN NaN NaN 255.0
44 global_heat_content_solid global_heat_content_solid global_heat_content_solid J NaN NaN NaN NaN 255.0
45 global_hfbasin global_hfbasin global northward ocean heat transport W NaN NaN NaN NaN 147.0
46 global_hfl global_hfl global implied heat transport W NaN NaN NaN NaN 147.0
47 global_moc global_moc global meridional overturning Sv NaN NaN NaN NaN 147.0
48 global_sltbasin global_sltbasin global northward ocean salt transport kg s-1 NaN NaN NaN NaN 147.0
49 global_wfl global_wfl global implied freshwater transport m3s-1 NaN NaN NaN NaN 147.0
50 gpsm geopotential_above_surface geopotential above surface m2 s-2 4.3.0 unstructured 1.0 NaN NaN
51 heat_content_liquid_water heat_content_liquid_water heat_content_liquid_water J m-2 NaN unstructured 1.0 NaN 255.0
52 heat_content_seaice heat_content_seaice heat_content_seaice J m-2 NaN unstructured 1.0 NaN 255.0
53 heat_content_snow heat_content_snow heat_conten_snow J m-2 NaN unstructured 1.0 NaN 255.0
54 heat_content_total heat_content_total heat_content_total J m-2 NaN unstructured 1.0 NaN 255.0
55 height_2_bnds NaN NaN NaN NaN NaN NaN NaN NaN
56 height_bnds NaN NaN NaN NaN NaN NaN NaN NaN
57 hfls lhflx latent heat flux W m-2 10.0.0 unstructured 1.0 NaN NaN
58 hfss shflx sensible heat flux W m-2 11.0.0 unstructured 1.0 NaN NaN
59 hi hi ice thickness m NaN unstructured 1.0 NaN 255.0
60 hs hs snow thickness m NaN unstructured 1.0 NaN 255.0
61 hus specific_humidity Specific humidity kg kg-1 0.1.0 unstructured 1.0 NaN NaN
62 hydro_discharge_box discharge local discharge kg m-2 s-1 NaN unstructured 1.0 NaN 255.0
63 hydro_discharge_ocean_box discharge_ocean discharge to the ocean m3 s-1 NaN unstructured 1.0 NaN 255.0
64 hydro_drainage_box drainage drainage kg m-2 s-1 NaN unstructured 1.0 NaN 255.0
65 hydro_fract_snow_box fraction of snow on surface NaN - 202.0.1 unstructured 1.0 NaN NaN
66 hydro_fract_water_box fraction of water on surface surface_wet_fraction NaN - 201.0.1 unstructured 1.0 NaN NaN
67 hydro_q_snocpymlt_box heating_snow_cpy_melt NaN W m-2 NaN unstructured 1.0 NaN 255.0
68 hydro_runoff_box surface_runoff Surface runoff kg m-2 s-1 NaN unstructured 1.0 NaN 255.0
69 hydro_transpiration_box surface_transpiration Transpiration from surface kg m-2 s-1 NaN unstructured 1.0 NaN 255.0
70 hydro_w_ice_sl_box Ice content in soil layers NaN m water equivalent NaN unstructured 1.0 NaN 255.0
71 hydro_w_skin_box skin_reservoir Water content in skin reservoir of surface m water equivalent 211.0.1 unstructured 1.0 NaN NaN
72 hydro_w_snow_box Water content of snow reservoir on surface NaN m water equivalent 212.0.1 unstructured 1.0 NaN NaN
73 hydro_w_soil_column_box Water content in the whole soil column NaN m water equivalent 213.0.1 unstructured 1.0 NaN NaN
74 hydro_w_soil_sl_box Water content in soil layers NaN m water equivalent NaN unstructured 1.0 NaN 255.0
75 ice_extent_nh ice_extent_nh ice_extent_nh km^2 NaN NaN NaN NaN 255.0
76 ice_extent_sh ice_extent_sh ice_extent_sh km^2 NaN NaN NaN NaN 255.0
77 ice_u ice_u zonal velocity m/s NaN unstructured 1.0 NaN 255.0
78 ice_v ice_v meridional velocity m/s NaN unstructured 1.0 NaN 255.0
79 ice_volume_nh ice_volume_nh ice_volume_nh km^3 NaN NaN NaN NaN 255.0
80 ice_volume_sh ice_volume_sh ice_volume_sh km^3 NaN NaN NaN NaN 255.0
81 kin_energy_global kin_energy kin_energy J NaN NaN NaN NaN 255.0
82 mlotst mlotst ocean_mixed_layer_thickness_defined_by_sigma_t m NaN unstructured 1.0 NaN 255.0
83 pacific_hfbasin pacific_hfbasin indopacific northward ocean heat transport W NaN NaN NaN NaN 149.0
84 pacific_hfl pacific_hfl indopacific implied heat transport W NaN NaN NaN NaN 149.0
85 pacific_moc pacific_moc indopacific meridional overturning Sv NaN NaN NaN NaN 149.0
86 pacific_sltbasin pacific_sltbasin indopacific northward ocean salt transport kg s-1 NaN NaN NaN NaN 149.0
87 pacific_wfl pacific_wfl indopacific implied freshwater transport W NaN NaN NaN NaN 149.0
88 pfull air_pressure Pressure Pa 0.3.0 unstructured 1.0 NaN NaN
89 pot_energy_global pot_energy pot_energy J NaN NaN NaN NaN 255.0
90 potential_enstrophy_global potential_enstrophy potential_enstrophy NaN NaN NaN NaN NaN 255.0
91 pr pr precipitation flux kg m-2 s-1 52.1.0 unstructured 1.0 NaN NaN
92 prlr prlr large-scale precipitation flux (water) kg m-2 s-1 77.1.0 unstructured 1.0 NaN NaN
93 prls prls large-scale precipitation flux (snow) kg m-2 s-1 59.1.0 unstructured 1.0 NaN NaN
94 prw total_vapour vertically integrated water vapour kg m-2 64.1.0 unstructured 1.0 atmosphere NaN
95 ps surface_air_pressure surface pressure Pa 0.3.0 unstructured 1.0 NaN NaN
96 psl mean sea level pressure mean sea level pressure Pa 1.3.0 unstructured 1.0 NaN NaN
97 qgvi total_graupel vertically integrated graupel kg m-2 223.1.0 unstructured 1.0 atmosphere NaN
98 qrvi total_rain vertically integrated rain kg m-2 221.1.0 unstructured 1.0 atmosphere NaN
99 qsvi total_snow vertically integrated snow kg m-2 222.1.0 unstructured 1.0 atmosphere NaN
100 rld downwelling_longwave_flux_in_air downwelling longwave radiation W m-2 3.5.0 unstructured 1.0 NaN NaN
101 rlds surface_downwelling_longwave_flux_in_air surface downwelling longwave radiation W m-2 3.5.0 unstructured 1.0 NaN NaN
102 rldscs surface_downwelling_longwave_flux_in_air_assuming_clear_sky surface downwelling clear-sky longwave radiation W m-2 203.5.0 unstructured 1.0 NaN NaN
103 rlu upwelling_longwave_flux_in_air upwelling longwave radiation W m-2 4.5.0 unstructured 1.0 NaN NaN
104 rlus surface_upwelling_longwave_flux_in_air surface upwelling longwave radiation W m-2 199.5.0 unstructured 1.0 NaN NaN
105 rlut toa_outgoing_longwave_flux toa outgoing longwave radiation W m-2 4.5.0 unstructured 1.0 toa NaN
106 rlutcs toa_outgoing_longwave_flux_assuming_clear_sky toa outgoing clear-sky longwave radiation W m-2 204.5.0 unstructured 1.0 toa NaN
107 rsd downwelling_shortwave_flux_in_air downwelling shortwave radiation W m-2 7.4.0 unstructured 1.0 NaN NaN
108 rsdoabsorb rsdoabsorb Net Rate of Absorption of Shortwave Energy in Ocean Layer W m-2 NaN unstructured 1.0 NaN 255.0
109 rsds surface_downwelling_shortwave_flux_in_air surface downwelling shortwave radiation W m-2 7.4.0 unstructured 1.0 NaN NaN
110 rsdscs surface_downwelling_shortwave_flux_in_air_assuming_clear_sky surface downwelling clear-sky shortwave radiation W m-2 207.4.0 unstructured 1.0 NaN NaN
111 rsdt toa_incoming_shortwave_flux toa incident shortwave radiation W m-2 201.4.0 unstructured 1.0 toa NaN
112 rsu upwelling_shortwave_flux_in_air upwelling shortwave radiation W m-2 8.4.0 unstructured 1.0 NaN NaN
113 rsus surface_upwelling_shortwave_flux_in_air surface upwelling shortwave radiation W m-2 199.4.0 unstructured 1.0 NaN NaN
114 rsuscs surface_upwelling_shortwave_flux_in_air_assuming_clear_sky surface upwelling clear-sky shortwave radiation W m-2 209.4.0 unstructured 1.0 NaN NaN
115 rsut toa_outgoing_shortwave_flux toa outgoing shortwave radiation W m-2 8.4.0 unstructured 1.0 toa NaN
116 rsutcs toa_outgoing_shortwave_flux_assuming_clear_sky toa outgoing clear-sky shortwave radiation W m-2 208.4.0 unstructured 1.0 toa NaN
117 sea_level_pressure Sea_Level_Pressure Sea Level Pressure Pa NaN unstructured 1.0 NaN 255.0
118 sfcwind sfcwind 10m windspeed m s-1 1.2.0 unstructured 1.0 NaN NaN
119 sic sea_ice_cover fraction of ocean covered by sea ice NaN 0.2.10 unstructured 1.0 NaN NaN
120 sit siced sea ice thickness m 1.2.10 unstructured 1.0 NaN NaN
121 so sea_water_salinity sea water salinity psu NaN unstructured 1.0 NaN 5.0
122 soil_depth_energy_bnds NaN NaN NaN NaN NaN NaN NaN NaN
123 soil_depth_water_bnds NaN NaN NaN NaN NaN NaN NaN NaN
124 sse_grnd_hflx_old_box grnd_hflx_old Ground heat flux (old) J m-2 s-1 NaN unstructured 1.0 NaN 255.0
125 sse_hcap_grnd_old_box heat_capacity_ground_old Ground heat capacity (old) J m-2 K-1 NaN unstructured 1.0 NaN 255.0
126 sse_t_soil_sl_box soil_temperature NaN K NaN unstructured 1.0 NaN 255.0
127 ssh_global ssh_global ssh_global m NaN NaN NaN NaN 255.0
128 sss_global sss_global global mean sea surface salinity m NaN NaN NaN NaN 255.0
129 sst_global sst_global global mean sea surface temperature m NaN NaN NaN NaN 255.0
130 ta air_temperature Temperature K 0.0.0 unstructured 1.0 NaN NaN
131 tas tas temperature in 2m K 0.0.0 unstructured 1.0 NaN NaN
132 tauu u_stress u-momentum flux at the surface N m-2 17.2.0 unstructured 1.0 NaN NaN
133 tauv v_stress v-momentum flux at the surface N m-2 18.2.0 unstructured 1.0 NaN NaN
134 tke tke turbulent kinetic energy m2 s-2 NaN unstructured 1.0 NaN 255.0
135 to sea_water_potential_temperature sea water potential temperature deg C NaN unstructured 1.0 NaN 2.0
136 total_energy_global total_energy total_energy J NaN NaN NaN NaN 255.0
137 total_salt total_salt total_salt kg NaN NaN NaN NaN 255.0
138 total_saltinliquidwater total_saltinliquidwater total_saltinliquidwater kg NaN NaN NaN NaN 255.0
139 total_saltinseaice total_saltinseaice total_saltinseaice kg NaN NaN NaN NaN 255.0
140 totalsnowfall_global totalsnowfall totalsnowfall m/s NaN NaN NaN NaN 255.0
141 ts surface_temperature surface temperature K 0.0.0 unstructured 1.0 NaN NaN
142 turb_fact_q_air_box fact_q_air NaN NaN NaN unstructured 1.0 NaN 255.0
143 turb_fact_qsat_srf_box fact_qsat_srf NaN NaN NaN unstructured 1.0 NaN 255.0
144 turb_fact_qsat_trans_srf_box fact_qsat_trans_srf NaN NaN NaN unstructured 1.0 NaN 255.0
145 u u u zonal velocity component m/s NaN unstructured 1.0 NaN 255.0
146 ua eastward_wind Zonal wind m s-1 2.2.0 unstructured 1.0 NaN NaN
147 uas uas zonal wind in 10m m s-1 2.2.0 unstructured 1.0 NaN NaN
148 v v v meridional velocity component m/s NaN unstructured 1.0 NaN 255.0
149 va northward_wind Meridional wind m s-1 3.2.0 unstructured 1.0 NaN NaN
150 vas vas meridional wind in 10m m s-1 3.2.0 unstructured 1.0 NaN NaN
151 verticallyTotal_mass_flux_e verticallyTotal_mass_flux_e vertically integrated mass flux at edges NaN NaN unstructured 3.0 NaN 255.0
152 vor relative_vorticity_on_cells Vorticity s-1 12.2.0 unstructured 1.0 NaN NaN
153 vort vort vorticity 1/s NaN unstructured 2.0 NaN 255.0
154 w w vertical velocity at cells m/s NaN unstructured 1.0 NaN 255.0
155 wa upward_air_velocity Vertical velocity m s-1 9.2.0 unstructured 1.0 NaN NaN
156 wap omega vertical velocity Pa s-1 8.2.0 unstructured 1.0 NaN NaN
157 zg geometric_height_at_full_level_center geometric height at full level center m 6.3.0 unstructured 1.0 NaN NaN
158 zos zos.TL2 surface elevation at cell center m NaN unstructured 1.0 NaN 1.0
[ ]: