% metaADVExample - an example script for processing a single ADV data file.  
% The steps here follow the steps outlined in the manual
%
% you should be running this script from the directory in which the data
% files reside
%
% modified to read global attributes from a common file 9/14/07 etm
%
% filename must contain a list of Global attributes 


%%% START USGS BOILERPLATE -------------
% These programs are intened for us in adjusting metadata terms in 
% netCDF files from the USGS CMGP Oceanographic time-series data 
% archive
%
% Program written Matlab 7.6.0.342 (R2008a)
% Program ran on Linux PC with RHEL4 and on Windows XP PC.
%
% "Although this program has been used by the USGS, no warranty, 
% expressed or implied, is made by the USGS or the United States 
% Government as to the accuracy and functioning of the program 
% and related program material nor shall the fact of distribution 
% constitute any such warranty, and no responsibility is assumed 
% by the USGS in connection therewith."
%%% END USGS BOILERPLATE --------------

   gatt=read_globalatts(fileName);  % provides values to insert below

% Written by Marinna Martini for the U.S. Geological Survey 
% Coastal & Marine Program Woods Hole Field Center, Woods Hole, MA
% http://woodshole.er.usgs.gov/ Please report bugs to mmartini@usgs.gov

% ------------- Step I what are the file names?
% processing file for 7772001.adr
adrFile = '7772001.adr';
if isnumeric(gatt.MOORING)
    gatt.MOORING=num2str(gatt.MOORING);
end
if length(gatt.MOORING) < 4
    gatt.MOORING=[gatt.MOORING adrFile(4)];
end

% ------------- Step II what is the metadata?
%
% here is the metadata structure for this deployment
%  - if you use an empty char string denoted by '' for an external sensor serial
%    number, then that data will not be included
%  - calibration info will be assigned to variable attributes dynamically
%    based on the field names of items under the struct field cals.
% Do not nest more structs under cals, or adr2cdf will not record the data
%
% no two data loggers can have the same four digit mooring number
%four character value MMML, MMM mooring + L location
metadata.mooring_number = [num2str(gatt.MOORING) '2']; 
metadata.deployment_start = gatt.Deployment_date; % the in water time
metadata.deployment_end = gatt.Recovery_date; % the out of water time
metadata.lon = gatt.longitude; % decimal degrees, West = negative
metadata.lat = gatt.latitude; % decimal degrees, South = negative
metadata.declination = gatt.magnetic_variation; % degrees west is negative
metadata.water_depth = gatt.WATER_DEPTH; % m (DO NOT OMIT!)
%
% these are logger specific
metadata.system.serial = 'G345'; % serial number of the Hydra electronics
metadata.system.synch_to = 'G357'; % G### | nothing
metadata.adv.serial = 'B227';
metadata.adv.height = 1.203; % height of emitter above bottom, m
                       % used to reference anything coming from the ADV probe
% distance from emitter to sample volume: 18 cm for ADVO, 10 cm for ADVF 10 MHz, 5 cm for ADVF 16 MHz                           
metadata.adv.sample_volume_offset = 18; % cm
% for attached sensors, use '' for the serial number if there is no sensor, 
% and variable will not be written
metadata.extpress.serial = 'P68022'; % use the druck file name without the extension
metadata.extpress.height = 2.14; % m, same as adv if in probe
% pressure sensor calibration data will be read from the .drk file
% now add the external sensors, in this example, the OBS was on channel 1
% and the trans on channel 2
% anything under the metadat.ext#.cals structure gets written as an attibute
% in the netCDF file
metadata.ext1.serial = ''; % OBS### | TRANS### | XXX####
metadata.ext1.height = 0; % m
metadata.ext1.cals.gain = .33; % for OBS gain setting, g/l per volt
% if coef is present, dohydraoptic will apply the coefficients to OBS data
% use [0 1 0], [] or omit if there is no calibration for concentration
metadata.ext1.cals.coef = []; % coefficients to a calibration eqation
% more than three coefficients will be accommodated as higher powers.
% these are not used but are informative
%metadata.ext1.cals.units = 'kg/m-3'; % or g/l
%metadata.ext1.cals.equation = 'Conc [kg/m-3] = coef(1)+coef(2)*V+coef(3)*V^2';
% anything under the metadat.ext#.cals structure gets written as an attibute
% in the netCDF file
metadata.ext2.serial = 'OBS2183'; % OBS### | TRANS### | XXX####
metadata.ext2.height = 0.98; % m
metadata.ext2.cals.gain = .33; % for OBS gain setting, g/l per volt
% if coef is present, dohydraoptic will apply the coefficients to OBS data
% use [0 1 0], [] or omit if there is no calibration for concentration
metadata.ext2.cals.coef = []; % coefficients to a calibration eqation
% the following are required for trans data
metadata.CTD.serial = ''; % 
metadata.CTD.height = 0; % m
% SBE37's record data in real units and do not need applied cals
metadata.metafile = mfilename('fullpath');
metadata.metafile_date = datestr(now);
metadata.metafile_version = '0.0';
metadata.metafile_author = 'MM/EM';
% if the following fields are missing, UNKOWN will be used
metadata.origin = gatt.DATA_ORIGIN; % with collaborator's data, could be USC, etc.
metadata.experiment = gatt.EXPERIMENT; 
metadata.description = gatt.DESCRIPTION; 
metadata.conventions = gatt.Conventions; 
metadata.project = gatt.PROJECT; % might also use OFA funding agency, such as MWRA, EPA, WCMG
metadata.whichscheme = 1; % which sampling scheme to read out
% always 1 for single sampling scheme data, not more than 3 

% omit bursts prior to deployment and after recovery when the instrument was in the water
% either process the first and last 300 to find limits of good data 
% to process all data use : metadata.goodbursts = [1 Inf];
metadata.goodbursts = [207 2839];

% set the root name for the output files
outFileRoot = [gatt.MOORING 'adv']; 

% these programs take a long time to run, so keep a diary
diary(sprintf('run%s',datestr(now,30))) 

% change the 0's to 1's sequentially for the steps below to work through
% the processing.  Most of the programs take several hours to run on 30K bursts, so
% testing the threshold's used on a small subset of the data is encouraged
% before processing everything.

% ---------- Step III convert from vendor's raw binary to netCDF & get quality info
if 0,
    % now run the file to convert from raw to netcdf
    diagnostics.pcountout = 0;      % both should be 0 !!!
    diagnostics.fixpfreq = 0;
    adr2cdf(adrFile,outFileRoot,metadata.goodbursts,metadata.goodbursts,diagnostics,metadata);
end

% ---------- Step IV semi-automated cleaning
%  saving each step with a sequential makes sense at the moment- 
%  depending on the data, doing just thumbfinger may  be good, but followint
%  with  deglitchvector may clear things up further, 
%  play around and see what's right for your data
%
if 0,
    % make a copy of the files, fixbadadv works directly on them
    % use the already canged *s1.cdf to be changed further
    disp(sprintf('copying %s to %s',[outFileRoot 'b.cdf'], [outFileRoot 'b1.cdf']))
    copyfile([outFileRoot 'b.cdf'], [outFileRoot 'b1.cdf'])
    disp(sprintf('copying %s to %s',[outFileRoot 's.cdf'], [outFileRoot 's1.cdf']))
    copyfile([outFileRoot 's.cdf'], [outFileRoot 's1.cdf'])
    disp(sprintf('copying %s to %s',[outFileRoot 'q.cdf'], [outFileRoot 'q1.cdf']))
    copyfile([outFileRoot 'q.cdf'], [outFileRoot 'q1.cdf'])
end

if 0,
    % function cleanhydra(cdfbFile, cdfsFile, cdfqFile, operation, ...
    %       'variables', {vnames}, 'settings', settings.n, ...
    %       'burstrange', [1:Inf]) 
    % the process below is reasonably robust over multiple data sets
    % to run thumbfinger with std of 10, use this:
      seta.nsd=10;
       cleanhydra([outFileRoot 'b1.cdf'], [outFileRoot 's1.cdf'], ...
        [outFileRoot 'q1.cdf'], 'thumbfinger','settings',seta);
    % make a copy of the files, fixbadadv works directly on them
    % use the already canged *s1.cdf to be changed further
    disp(sprintf('copying %s to %s',[outFileRoot 'b1.cdf'], [outFileRoot 'b2.cdf']))
    copyfile([outFileRoot 'b1.cdf'], [outFileRoot 'b2.cdf'])
    disp(sprintf('copying %s to %s',[outFileRoot 's1.cdf'], [outFileRoot 's2.cdf']))
    copyfile([outFileRoot 's1.cdf'], [outFileRoot 's2.cdf'])
    disp(sprintf('copying %s to %s',[outFileRoot 'q1.cdf'], [outFileRoot 'q2.cdf']))
    copyfile([outFileRoot 'q1.cdf'], [outFileRoot 'q2.cdf'])

    % to run just deglitch1vector with nsd=2.8, use:
      seta.nsd=2.8;
       cleanhydra([outFileRoot 'b2.cdf'], [outFileRoot 's2.cdf'], ...
        [outFileRoot 'q2.cdf'], 'deglitch1vector','settings', seta);
 
    % to modify the default settings, and just do bursts 1361-1365, use this syntax:
    %  seta.nsd=3; seta.ndt=10;
    %   cleanhydra([outFileRoot 'b1.cdf'], [outFileRoot 's1.cdf'], [outFileRoot 'q11.cdf'], ...
    %    'deglitch1vector','settings',seta,'burstrange', [1361:1365]);

    %  This is VERY handy- take note!
    %  replace the first 5 values of pressure with the mean of the rest of the burst
    %  rm_badsamp can operate on ANY variable in the file, extsensors etc
    %    seta.var='pres'; seta.rvalue='mean'; seta.samps=[1:5];
    %   cleanhydra([outFileRoot 'b1.cdf'], [outFileRoot 's1.cdf'], [outFileRoot 'q1.cdf'], ...
    %    'rm_badsamp','settings',seta);

    % use of autoclean (does thumbfinger, corrcheck, then deglitch all using nsd=2.8)
    %  is possible, but not recommended because the results are awful
    %   cleanhydra([outFileRoot 'b1.cdf'], [outFileRoot 's1.cdf'], ...
    %    [outFileRoot 'q1.cdf'], 'autoclean');
    
end

% ------------ Step IVb fix v and b ranges, if desired
%  If vrange and brange are noisy- this is the place to fix it- harder to
%  do after the .nc version is created by adv2nc.
%  see example_fix_vbrange.m for what to do, then add what was done here.
%  determining the parameters usually requires several tries to get right.
%
if 0
    seta.std_threshold=10;
    seta.min=300; seta.max=450;
    seta.npts=25;
    seta.nstds=2.4;
    cleanhydra([outFileRoot 'b3.cdf'], [outFileRoot 's3.cdf'], ...
        [outFileRoot 'q3.cdf'], 'fix_vbrange','settings', seta, ...
        'variables',{'vrange'});
    
    seta.std_threshold=10;
    seta.min=400; seta.max=600;
    seta.npts=25;
    seta.nstds=2.4;
    cleanhydra([outFileRoot 'b3.cdf'], [outFileRoot 's3.cdf'], ...
        [outFileRoot 'q3.cdf'], 'fix_vbrange','settings', seta, ...
        'variables',{'brange'});
end

% ------------ Step V interactive flagging
%
%  This instrument got +/- 100 vels in storms, which have very high std in
%  those periods, but the burst data looks fine-  scutoff of 40, 55 and 9
%  were used to allow all bursts.
%  ** for other datasets lower thresholds are preferable **
if 0,
    % settings.currentBeam is 1 by default
    seta.currentBeam=1;   seta.new_scutoff= [40 40 40 40 40]; % match values from beam 1
      flagbadadv('QualityFile', [outFileRoot 'q3.cdf'], 'settings',seta)
    seta.currentBeam=2;   seta.new_scutoff= [55 55 55 55 55]; % match values from beam 1
      flagbadadv('QualityFile', [outFileRoot 'q3.cdf'], 'settings',seta)
    seta.currentBeam=3;   seta.new_scutoff= [9 9 9 9 9]; % match values from beam 1 & 2
      flagbadadv('QualityFile', [outFileRoot 'q3.cdf'], 'settings',seta)
end

if 0,
    % fixbadadv(dirtybFile, dirtysFile, Qdata)
    fixbadadvvel([outFileRoot 'b3.cdf'],[outFileRoot 's3.cdf'],[outFileRoot 'q3.cdf']);
end

% ---------- Step VI convert to scientific units and apply rotations
if 0,
    % another time consuming step, so keep a diary
    diary(sprintf('run%s',datestr(now,30)))
    % unless otherwise set...
    % switches.ecorr_showplots = 0;
    % deglitch has been removed... use cleanhydra
    % despike has been removed... use cleanhydra
    % orientation overrides left intact in adv2nc
    % Vector length must match the number of bursts this function is
    %  asked to process
    % switches.override_heading = [];
    % switches.override_pitch = [];
    % switches.override_roll = [];
    % pressure mask left intact in adv2nc
    % switches.override_pmask = []; % pressure spike mask
    % example to omit the first 5 samples: 
    %    switches.override_pmask = [zeros(5,1); ones(4500-5,1)];
    switches = []; % invoke all defaults

    Adv = adv2nc([outFileRoot 'b3.cdf'], [outFileRoot 's3.cdf'], outFileRoot, switches, [207 2839]);
% this run took 415 minutes, so don't expect fast completion!
end

diary off