{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Merge multiple datasets\n\nExample to merge multiple data products into one using ACT.\nShows how to adjust the timestamp if the timestamps are at\ndifferent part of the sample interval (left, right, center).\nAlso shows how to apply QC information, merge and resample\ndata using xarray and plot/write out the results.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\nimport xarray as xr\nfrom arm_test_data import DATASETS\n\nimport act\n\n# Set data files\n# An alternative to this is to download data from the\n# ARM Data Webservice as shown in the discovery plot_neon.py example\nebbr_file = DATASETS.fetch('sgp30ebbrE13.b1.20190601.000000.nc')\necor_file = DATASETS.fetch('sgp30ecorE14.b1.20190601.000000.cdf')\nsebs_file = DATASETS.fetch('sgpsebsE14.b1.20190601.000000.cdf')\n\n# Read data into datasets\nds_ebbr = act.io.arm.read_arm_netcdf(ebbr_file, use_base_time=True)\nds_ecor = act.io.arm.read_arm_netcdf(ecor_file, use_base_time=True)\nds_sebs = act.io.arm.read_arm_netcdf(sebs_file, use_base_time=True)\n\n# Check for ARM DQRs and add them to the QC variables\nds_ebbr = act.qc.arm.add_dqr_to_qc(ds_ebbr)\nds_ecor = act.qc.arm.add_dqr_to_qc(ds_ecor)\nds_sebs = act.qc.arm.add_dqr_to_qc(ds_sebs)\n\n# The ECOR and EBBR have different definitions of latent heat\n# flux and what is positive vs negative.  Check out the ARM\n# Handbooks for more information\nds_ecor['lv_e'].values = ds_ecor['lv_e'].values * -1.0\n\n# For example purposes, let's rename the ecor latent heat flux\nds_ecor = ds_ecor.rename({'lv_e': 'latent_heat_flux_ecor'})\nds_ecor['latent_heat_flux_ecor'].attrs['ancillary_variables'] = 'qc_latent_heat_flux_ecor'\nds_ecor = ds_ecor.rename({'qc_lv_e': 'qc_latent_heat_flux_ecor'})\n\n# Also going to Switch some QC for example purposes\nqc = ds_ecor['qc_latent_heat_flux_ecor'].values\nqc[10:20] = 2\nds_ecor['qc_latent_heat_flux_ecor'].values = qc\n\n# There is a difference in how these timestamps are defined\n# The EBBR is at the end of the sampling interval and the\n# ECOR is at the beginning.  Knowing this, we can shift the\n# EBBR timestampes by 30 minutes to coincide with the ECOR\nds_ebbr = act.utils.datetime_utils.adjust_timestamp(ds_ebbr, offset=-30 * 60)\n\n# Now, we can merge all these datasets into one product\nds = xr.merge([ds_ecor, ds_ebbr, ds_sebs], compat='override')\n\n# Apply the QC information to set all flagged data to missing/NaN\nds.qcfilter.datafilter(\n    del_qc_var=False, rm_assessments=['Bad', 'Incorrect', 'Indeterminate', 'Suspect']\n)\n\n# Plot up data from the merged dataset for each of the instruments\ndisplay = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(3,))\ndisplay.plot('latent_heat_flux_ecor', label='ECOR', subplot_index=(0,))\ndisplay.plot('latent_heat_flux', label='EBBR', subplot_index=(0,))\nplt.legend()\ndisplay.plot('surface_soil_heat_flux_1', label='SEBS', subplot_index=(1,))\n\n# Plot out the QC information that was modified as well\ndisplay.qc_flag_block_plot('latent_heat_flux_ecor', subplot_index=(2,))\nplt.show()\n\n# Resample the data to 1 hour mean\n# Check out the xarray documentation for more information\n# on the resample function. Options include mean, median,\n# max, min, sum, nearest, and more.\nds = ds.resample(time='h').mean(keep_attrs=True)\n\n# Plot up data from the hourly merged dataset for ECOR and EBBR\ndisplay = act.plotting.TimeSeriesDisplay(ds, figsize=(15, 10), subplot_shape=(1,))\ndisplay.plot('latent_heat_flux_ecor', label='ECOR', subplot_index=(0,))\ndisplay.plot('latent_heat_flux', label='EBBR', subplot_index=(0,))\nplt.show()\n\n# Write data out to netcdf\nds.to_netcdf('./sgpecor_ebbr_sebs.nc')"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.14.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}