Robust Script for getting variable metadata

This is a fairly robust way of getting the details of the variables. We avoid the possible collisions in catalog merges by loading each file individually as an xarray dataset.

# basics
import intake
import numpy as np
import xarray as xr
import dask  # memory-efficient parallel computation and delayed execution (lazy evaluation).

# panadas for the tables
import pandas as pd

pd.set_option("display.max_rows", None)
%run gem_helpers.ipynb

Choose catalog:

catalog_file = "/work/ka1081/Catalogs/dyamond-nextgems.json"
my_simulation_id = "dpp0066"
cat = intake.open_esm_datastore(catalog_file).search(simulation_id=my_simulation_id)
cat

ICON-ESM catalog with 7 dataset(s) from 8114 asset(s):

	unique
variable_id	155
project	1
institution_id	1
source_id	1
experiment_id	1
simulation_id	1
realm	2
frequency	4
time_reduction	2
grid_label	1
level_type	3
time_min	406
time_max	406
grid_id	1
format	1
uri	8114

loading data:

times = get_from_cat(cat, "time_min").values[
    [0, -1]
]  # we assume that all files are present at the first or last timestep. Iterate over more, if you have less trust in the cleanness of the run
all_atts = []
for time in times:
    files = cat.search(time_min=time)
    for uri in get_from_cat(files, "uri")["uri"].values:
        data = xr.open_dataset(uri)
        all_atts.extend(
            dict(variable_id=var, **data[var].attrs) for var in data.data_vars
        )  # we take the detour via dict and ** to add the variable name
frame = pd.DataFrame(all_atts)
frame = frame.drop_duplicates().sort_values(list(frame)).reset_index(drop=True)
frame

	variable_id	standard_name	long_name	units	param	CDI_grid_type	number_of_grid_in_reference	level_type	code
0	A_tracer_v_to	A_tracer_v_to	sea water potential temperature(A_tracer_v)	kg/kg	NaN	unstructured	1.0	NaN	255.0
1	A_veloc_v	A_veloc_v	vertical velocity diffusion	kg/kg	NaN	unstructured	3.0	NaN	255.0
2	FrshFlux_Evaporation_global	FrshFlux_Evaporation	FrshFlux_Evaporation_global	m/s	NaN	NaN	NaN	NaN	255.0
3	FrshFlux_Precipitation_global	FrshFlux_Precipitation	FrshFlux_Precipitation	m/s	NaN	NaN	NaN	NaN	255.0
4	FrshFlux_Runoff_global	FrshFlux_Runoff	FrshFlux_Runoff	m/s	NaN	NaN	NaN	NaN	255.0
5	FrshFlux_SnowFall_global	FrshFlux_SnowFall	FrshFlux_SnowFall_global	m/s	NaN	NaN	NaN	NaN	255.0
6	FrshFlux_TotalIce_global	FrshFlux_TotalIce	FrshFlux_TotalIce	m/s	NaN	NaN	NaN	NaN	255.0
7	FrshFlux_TotalOcean_global	FrshFlux_TotalOcean	FrshFlux_TotalOcean	m/s	NaN	NaN	NaN	NaN	255.0
8	FrshFlux_VolumeIce_global	FrshFlux_VolumeIce	FrshFlux_VolumeIce	m/s	NaN	NaN	NaN	NaN	255.0
9	FrshFlux_VolumeTotal_global	FrshFlux_VolumeTotal	FrshFlux_VolumeTotal	m/s	NaN	NaN	NaN	NaN	255.0
10	HeatFlux_Total_global	HeatFlux_Total	HeatFlux_Total	W/m2	NaN	NaN	NaN	NaN	255.0
11	Qbot	Qbot	Conductive heat flux at ice-ocean interface	W/m^2	NaN	unstructured	1.0	NaN	255.0
12	Qtop	Qtop	Energy flux available for surface melting	W/m^2	NaN	unstructured	1.0	NaN	255.0
13	Wind_Speed_10m	Wind_Speed_10m	Wind Speed at 10m height	m/s	NaN	unstructured	1.0	NaN	255.0
14	amoc26n	amoc26n	amoc26n	Sv	NaN	NaN	NaN	NaN	255.0
15	atlantic_hfbasin	atlantic_hfbasin	atlantic northward ocean heat transport	W	NaN	NaN	NaN	NaN	148.0
16	atlantic_hfl	atlantic_hfl	atlantic implied heat transport	W	NaN	NaN	NaN	NaN	148.0
17	atlantic_moc	atlantic_moc	atlantic meridional overturning	Sv	NaN	NaN	NaN	NaN	148.0
18	atlantic_sltbasin	atlantic_sltbasin	atlantic northward ocean salt transport	kg s-1	NaN	NaN	NaN	NaN	148.0
19	atlantic_wfl	atlantic_wfl	atlantic implied freshwater transport	m3s-1	NaN	NaN	NaN	NaN	148.0
20	atmos_fluxes_FrshFlux_Evaporation	atmos_fluxes_FrshFlux_Evaporation	atmos_fluxes_FrshFlux_Evaporation	[m/s]	NaN	unstructured	1.0	NaN	255.0
21	atmos_fluxes_FrshFlux_Precipitation	atmos_fluxes_FrshFlux_Precipitation	atmos_fluxes_FrshFlux_Precipitation	[m/s]	NaN	unstructured	1.0	NaN	255.0
22	atmos_fluxes_FrshFlux_Runoff	atmos_fluxes_FrshFlux_Runoff	atmos_fluxes_FrshFlux_Runoff	[m/s]	NaN	unstructured	1.0	NaN	255.0
23	atmos_fluxes_FrshFlux_SnowFall	atmos_fluxes_FrshFlux_SnowFall	atmos_fluxes_FrshFlux_SnowFall	[m/s]	NaN	unstructured	1.0	NaN	255.0
24	atmos_fluxes_HeatFlux_Latent	atmos_fluxes_HeatFlux_Latent	atmos_fluxes_HeatFlux_Latent	[W/m2]	NaN	unstructured	1.0	NaN	255.0
25	atmos_fluxes_HeatFlux_LongWave	atmos_fluxes_HeatFlux_LongWave	atmos_fluxes_HeatFlux_LongWave	[W/m2]	NaN	unstructured	1.0	NaN	255.0
26	atmos_fluxes_HeatFlux_Sensible	atmos_fluxes_HeatFlux_Sensible	atmos_fluxes_HeatFlux_Sensible	[W/m2]	NaN	unstructured	1.0	NaN	255.0
27	atmos_fluxes_HeatFlux_ShortWave	atmos_fluxes_HeatFlux_ShortWave	atmos_fluxes_HeatFlux_ShortWave	[W/m2]	NaN	unstructured	1.0	NaN	255.0
28	atmos_fluxes_HeatFlux_Total	atmos_fluxes_HeatFlux_Total	atmos_fluxes_HeatFlux_Total	[W/m2]	NaN	unstructured	1.0	NaN	255.0
29	atmos_fluxes_stress_x	atmos_fluxes_stress_x	atmos_fluxes_stress_x	Pa	NaN	unstructured	1.0	NaN	255.0
30	atmos_fluxes_stress_xw	atmos_fluxes_stress_xw	atmos_fluxes_stress_xw	Pa	NaN	unstructured	1.0	NaN	255.0
31	atmos_fluxes_stress_y	atmos_fluxes_stress_y	atmos_fluxes_stress_y	Pa	NaN	unstructured	1.0	NaN	255.0
32	atmos_fluxes_stress_yw	atmos_fluxes_stress_yw	atmos_fluxes_stress_yw	Pa	NaN	unstructured	1.0	NaN	255.0
33	cl	cl	cloud area fraction	m2 m-2	22.6.0	unstructured	1.0	NaN	NaN
34	cli	cli	specific cloud ice content	kg kg-1	82.1.0	unstructured	1.0	NaN	NaN
35	clivi	total_cloud_ice	vertically integrated cloud ice	kg m-2	70.1.0	unstructured	1.0	atmosphere	NaN
36	cllvi	total_cloud_water	vertically integrated cloud water	kg m-2	69.1.0	unstructured	1.0	atmosphere	NaN
37	clt	clt	total cloud cover	m2 m-2	1.6.0	unstructured	1.0	NaN	NaN
38	clw	clw	specific cloud water content	kg kg-1	22.1.0	unstructured	1.0	NaN	NaN
39	conc	conc	ice concentration in each ice class	NaN	NaN	unstructured	1.0	NaN	255.0
40	cptgz	cptgz	dry static energy	m2 s-2	NaN	unstructured	1.0	NaN	255.0
41	cptgzvi	vertically integrated dry static energy	vert_int_dry_static_energy	m2 s-2	NaN	unstructured	1.0	atmosphere	255.0
42	evspsbl	evap	evaporation	kg m-2 s-1	6.1.0	unstructured	1.0	NaN	NaN
43	global_heat_content	global_heat_content	global_heat_content	J	NaN	NaN	NaN	NaN	255.0
44	global_heat_content_solid	global_heat_content_solid	global_heat_content_solid	J	NaN	NaN	NaN	NaN	255.0
45	global_hfbasin	global_hfbasin	global northward ocean heat transport	W	NaN	NaN	NaN	NaN	147.0
46	global_hfl	global_hfl	global implied heat transport	W	NaN	NaN	NaN	NaN	147.0
47	global_moc	global_moc	global meridional overturning	Sv	NaN	NaN	NaN	NaN	147.0
48	global_sltbasin	global_sltbasin	global northward ocean salt transport	kg s-1	NaN	NaN	NaN	NaN	147.0
49	global_wfl	global_wfl	global implied freshwater transport	m3s-1	NaN	NaN	NaN	NaN	147.0
50	gpsm	geopotential_above_surface	geopotential above surface	m2 s-2	4.3.0	unstructured	1.0	NaN	NaN
51	heat_content_liquid_water	heat_content_liquid_water	heat_content_liquid_water	J m-2	NaN	unstructured	1.0	NaN	255.0
52	heat_content_seaice	heat_content_seaice	heat_content_seaice	J m-2	NaN	unstructured	1.0	NaN	255.0
53	heat_content_snow	heat_content_snow	heat_conten_snow	J m-2	NaN	unstructured	1.0	NaN	255.0
54	heat_content_total	heat_content_total	heat_content_total	J m-2	NaN	unstructured	1.0	NaN	255.0
55	height_2_bnds	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
56	height_bnds	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
57	hfls	lhflx	latent heat flux	W m-2	10.0.0	unstructured	1.0	NaN	NaN
58	hfss	shflx	sensible heat flux	W m-2	11.0.0	unstructured	1.0	NaN	NaN
59	hi	hi	ice thickness	m	NaN	unstructured	1.0	NaN	255.0
60	hs	hs	snow thickness	m	NaN	unstructured	1.0	NaN	255.0
61	hus	specific_humidity	Specific humidity	kg kg-1	0.1.0	unstructured	1.0	NaN	NaN
62	hydro_discharge_box	discharge	local discharge	kg m-2 s-1	NaN	unstructured	1.0	NaN	255.0
63	hydro_discharge_ocean_box	discharge_ocean	discharge to the ocean	m3 s-1	NaN	unstructured	1.0	NaN	255.0
64	hydro_drainage_box	drainage	drainage	kg m-2 s-1	NaN	unstructured	1.0	NaN	255.0
65	hydro_fract_snow_box	fraction of snow on surface	NaN	-	202.0.1	unstructured	1.0	NaN	NaN
66	hydro_fract_water_box	fraction of water on surface surface_wet_fraction	NaN	-	201.0.1	unstructured	1.0	NaN	NaN
67	hydro_q_snocpymlt_box	heating_snow_cpy_melt	NaN	W m-2	NaN	unstructured	1.0	NaN	255.0
68	hydro_runoff_box	surface_runoff	Surface runoff	kg m-2 s-1	NaN	unstructured	1.0	NaN	255.0
69	hydro_transpiration_box	surface_transpiration	Transpiration from surface	kg m-2 s-1	NaN	unstructured	1.0	NaN	255.0
70	hydro_w_ice_sl_box	Ice content in soil layers	NaN	m water equivalent	NaN	unstructured	1.0	NaN	255.0
71	hydro_w_skin_box	skin_reservoir	Water content in skin reservoir of surface	m water equivalent	211.0.1	unstructured	1.0	NaN	NaN
72	hydro_w_snow_box	Water content of snow reservoir on surface	NaN	m water equivalent	212.0.1	unstructured	1.0	NaN	NaN
73	hydro_w_soil_column_box	Water content in the whole soil column	NaN	m water equivalent	213.0.1	unstructured	1.0	NaN	NaN
74	hydro_w_soil_sl_box	Water content in soil layers	NaN	m water equivalent	NaN	unstructured	1.0	NaN	255.0
75	ice_extent_nh	ice_extent_nh	ice_extent_nh	km^2	NaN	NaN	NaN	NaN	255.0
76	ice_extent_sh	ice_extent_sh	ice_extent_sh	km^2	NaN	NaN	NaN	NaN	255.0
77	ice_u	ice_u	zonal velocity	m/s	NaN	unstructured	1.0	NaN	255.0
78	ice_v	ice_v	meridional velocity	m/s	NaN	unstructured	1.0	NaN	255.0
79	ice_volume_nh	ice_volume_nh	ice_volume_nh	km^3	NaN	NaN	NaN	NaN	255.0
80	ice_volume_sh	ice_volume_sh	ice_volume_sh	km^3	NaN	NaN	NaN	NaN	255.0
81	kin_energy_global	kin_energy	kin_energy	J	NaN	NaN	NaN	NaN	255.0
82	mlotst	mlotst	ocean_mixed_layer_thickness_defined_by_sigma_t	m	NaN	unstructured	1.0	NaN	255.0
83	pacific_hfbasin	pacific_hfbasin	indopacific northward ocean heat transport	W	NaN	NaN	NaN	NaN	149.0
84	pacific_hfl	pacific_hfl	indopacific implied heat transport	W	NaN	NaN	NaN	NaN	149.0
85	pacific_moc	pacific_moc	indopacific meridional overturning	Sv	NaN	NaN	NaN	NaN	149.0
86	pacific_sltbasin	pacific_sltbasin	indopacific northward ocean salt transport	kg s-1	NaN	NaN	NaN	NaN	149.0
87	pacific_wfl	pacific_wfl	indopacific implied freshwater transport	W	NaN	NaN	NaN	NaN	149.0
88	pfull	air_pressure	Pressure	Pa	0.3.0	unstructured	1.0	NaN	NaN
89	pot_energy_global	pot_energy	pot_energy	J	NaN	NaN	NaN	NaN	255.0
90	potential_enstrophy_global	potential_enstrophy	potential_enstrophy	NaN	NaN	NaN	NaN	NaN	255.0
91	pr	pr	precipitation flux	kg m-2 s-1	52.1.0	unstructured	1.0	NaN	NaN
92	prlr	prlr	large-scale precipitation flux (water)	kg m-2 s-1	77.1.0	unstructured	1.0	NaN	NaN
93	prls	prls	large-scale precipitation flux (snow)	kg m-2 s-1	59.1.0	unstructured	1.0	NaN	NaN
94	prw	total_vapour	vertically integrated water vapour	kg m-2	64.1.0	unstructured	1.0	atmosphere	NaN
95	ps	surface_air_pressure	surface pressure	Pa	0.3.0	unstructured	1.0	NaN	NaN
96	psl	mean sea level pressure	mean sea level pressure	Pa	1.3.0	unstructured	1.0	NaN	NaN
97	qgvi	total_graupel	vertically integrated graupel	kg m-2	223.1.0	unstructured	1.0	atmosphere	NaN
98	qrvi	total_rain	vertically integrated rain	kg m-2	221.1.0	unstructured	1.0	atmosphere	NaN
99	qsvi	total_snow	vertically integrated snow	kg m-2	222.1.0	unstructured	1.0	atmosphere	NaN
100	rld	downwelling_longwave_flux_in_air	downwelling longwave radiation	W m-2	3.5.0	unstructured	1.0	NaN	NaN
101	rlds	surface_downwelling_longwave_flux_in_air	surface downwelling longwave radiation	W m-2	3.5.0	unstructured	1.0	NaN	NaN
102	rldscs	surface_downwelling_longwave_flux_in_air_assuming_clear_sky	surface downwelling clear-sky longwave radiation	W m-2	203.5.0	unstructured	1.0	NaN	NaN
103	rlu	upwelling_longwave_flux_in_air	upwelling longwave radiation	W m-2	4.5.0	unstructured	1.0	NaN	NaN
104	rlus	surface_upwelling_longwave_flux_in_air	surface upwelling longwave radiation	W m-2	199.5.0	unstructured	1.0	NaN	NaN
105	rlut	toa_outgoing_longwave_flux	toa outgoing longwave radiation	W m-2	4.5.0	unstructured	1.0	toa	NaN
106	rlutcs	toa_outgoing_longwave_flux_assuming_clear_sky	toa outgoing clear-sky longwave radiation	W m-2	204.5.0	unstructured	1.0	toa	NaN
107	rsd	downwelling_shortwave_flux_in_air	downwelling shortwave radiation	W m-2	7.4.0	unstructured	1.0	NaN	NaN
108	rsdoabsorb	rsdoabsorb	Net Rate of Absorption of Shortwave Energy in Ocean Layer	W m-2	NaN	unstructured	1.0	NaN	255.0
109	rsds	surface_downwelling_shortwave_flux_in_air	surface downwelling shortwave radiation	W m-2	7.4.0	unstructured	1.0	NaN	NaN
110	rsdscs	surface_downwelling_shortwave_flux_in_air_assuming_clear_sky	surface downwelling clear-sky shortwave radiation	W m-2	207.4.0	unstructured	1.0	NaN	NaN
111	rsdt	toa_incoming_shortwave_flux	toa incident shortwave radiation	W m-2	201.4.0	unstructured	1.0	toa	NaN
112	rsu	upwelling_shortwave_flux_in_air	upwelling shortwave radiation	W m-2	8.4.0	unstructured	1.0	NaN	NaN
113	rsus	surface_upwelling_shortwave_flux_in_air	surface upwelling shortwave radiation	W m-2	199.4.0	unstructured	1.0	NaN	NaN
114	rsuscs	surface_upwelling_shortwave_flux_in_air_assuming_clear_sky	surface upwelling clear-sky shortwave radiation	W m-2	209.4.0	unstructured	1.0	NaN	NaN
115	rsut	toa_outgoing_shortwave_flux	toa outgoing shortwave radiation	W m-2	8.4.0	unstructured	1.0	toa	NaN
116	rsutcs	toa_outgoing_shortwave_flux_assuming_clear_sky	toa outgoing clear-sky shortwave radiation	W m-2	208.4.0	unstructured	1.0	toa	NaN
117	sea_level_pressure	Sea_Level_Pressure	Sea Level Pressure	Pa	NaN	unstructured	1.0	NaN	255.0
118	sfcwind	sfcwind	10m windspeed	m s-1	1.2.0	unstructured	1.0	NaN	NaN
119	sic	sea_ice_cover	fraction of ocean covered by sea ice	NaN	0.2.10	unstructured	1.0	NaN	NaN
120	sit	siced	sea ice thickness	m	1.2.10	unstructured	1.0	NaN	NaN
121	so	sea_water_salinity	sea water salinity	psu	NaN	unstructured	1.0	NaN	5.0
122	soil_depth_energy_bnds	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
123	soil_depth_water_bnds	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
124	sse_grnd_hflx_old_box	grnd_hflx_old	Ground heat flux (old)	J m-2 s-1	NaN	unstructured	1.0	NaN	255.0
125	sse_hcap_grnd_old_box	heat_capacity_ground_old	Ground heat capacity (old)	J m-2 K-1	NaN	unstructured	1.0	NaN	255.0
126	sse_t_soil_sl_box	soil_temperature	NaN	K	NaN	unstructured	1.0	NaN	255.0
127	ssh_global	ssh_global	ssh_global	m	NaN	NaN	NaN	NaN	255.0
128	sss_global	sss_global	global mean sea surface salinity	m	NaN	NaN	NaN	NaN	255.0
129	sst_global	sst_global	global mean sea surface temperature	m	NaN	NaN	NaN	NaN	255.0
130	ta	air_temperature	Temperature	K	0.0.0	unstructured	1.0	NaN	NaN
131	tas	tas	temperature in 2m	K	0.0.0	unstructured	1.0	NaN	NaN
132	tauu	u_stress	u-momentum flux at the surface	N m-2	17.2.0	unstructured	1.0	NaN	NaN
133	tauv	v_stress	v-momentum flux at the surface	N m-2	18.2.0	unstructured	1.0	NaN	NaN
134	tke	tke	turbulent kinetic energy	m2 s-2	NaN	unstructured	1.0	NaN	255.0
135	to	sea_water_potential_temperature	sea water potential temperature	deg C	NaN	unstructured	1.0	NaN	2.0
136	total_energy_global	total_energy	total_energy	J	NaN	NaN	NaN	NaN	255.0
137	total_salt	total_salt	total_salt	kg	NaN	NaN	NaN	NaN	255.0
138	total_saltinliquidwater	total_saltinliquidwater	total_saltinliquidwater	kg	NaN	NaN	NaN	NaN	255.0
139	total_saltinseaice	total_saltinseaice	total_saltinseaice	kg	NaN	NaN	NaN	NaN	255.0
140	totalsnowfall_global	totalsnowfall	totalsnowfall	m/s	NaN	NaN	NaN	NaN	255.0
141	ts	surface_temperature	surface temperature	K	0.0.0	unstructured	1.0	NaN	NaN
142	turb_fact_q_air_box	fact_q_air	NaN	NaN	NaN	unstructured	1.0	NaN	255.0
143	turb_fact_qsat_srf_box	fact_qsat_srf	NaN	NaN	NaN	unstructured	1.0	NaN	255.0
144	turb_fact_qsat_trans_srf_box	fact_qsat_trans_srf	NaN	NaN	NaN	unstructured	1.0	NaN	255.0
145	u	u	u zonal velocity component	m/s	NaN	unstructured	1.0	NaN	255.0
146	ua	eastward_wind	Zonal wind	m s-1	2.2.0	unstructured	1.0	NaN	NaN
147	uas	uas	zonal wind in 10m	m s-1	2.2.0	unstructured	1.0	NaN	NaN
148	v	v	v meridional velocity component	m/s	NaN	unstructured	1.0	NaN	255.0
149	va	northward_wind	Meridional wind	m s-1	3.2.0	unstructured	1.0	NaN	NaN
150	vas	vas	meridional wind in 10m	m s-1	3.2.0	unstructured	1.0	NaN	NaN
151	verticallyTotal_mass_flux_e	verticallyTotal_mass_flux_e	vertically integrated mass flux at edges	NaN	NaN	unstructured	3.0	NaN	255.0
152	vor	relative_vorticity_on_cells	Vorticity	s-1	12.2.0	unstructured	1.0	NaN	NaN
153	vort	vort	vorticity	1/s	NaN	unstructured	2.0	NaN	255.0
154	w	w	vertical velocity at cells	m/s	NaN	unstructured	1.0	NaN	255.0
155	wa	upward_air_velocity	Vertical velocity	m s-1	9.2.0	unstructured	1.0	NaN	NaN
156	wap	omega	vertical velocity	Pa s-1	8.2.0	unstructured	1.0	NaN	NaN
157	zg	geometric_height_at_full_level_center	geometric height at full level center	m	6.3.0	unstructured	1.0	NaN	NaN
158	zos	zos.TL2	surface elevation at cell center	m	NaN	unstructured	1.0	NaN	1.0