% LISST100Xlog2cdf convert LISST.EXE raw ascii output to netCDF raw % % function lisst = LISST100Xlog2cdf(LISSTroot, ncroot, metadata) % LISSTroot = root file name for the LISST.EXE or LISSTST.EXE output % will seek the raw .log files only % note that Sequoia's software removes the headers from settling % experiments % The logfile columns are % 1:32 Light intensity on detectors 1 through 32 % 33 Laser transmission Sensor % 34 Battery voltage in raw counts % 35 External Auxiliary input 1 (0 to 5V = 0 to 4096) % 36 Laser Reference sensor % 37 Pressure in raw counts % 38 Temperature in units of 100ths of degrees C % 39 (Day*100 + Hour) at which data taken % 40 (Minutes*100 + Seconds) at which data taken % cdfroot = root file name for converted netCDF data % raw data will go to a .cdf file as a time series % there must be a deployment start date!! % metadata.deployment_start = gatt.Deployment_date; % the in water time % metadata.deployment_end = gatt.Recovery_date; % the out of water time % metadata.lon = gatt.longitude; % decimal degrees, West = negative % metadata.lat = gatt.latitude; % decimal degrees, South = negative % metadata.WATER_DEPTH = gatt.WATER_DEPTH; % m (DO NOT OMIT!) % % LISST specific information % metadata.serial = '1172'; % serial number of the LISST % metadata.type = 'C'; % LISST type (indicating size range) % metadata.zscFile = '1172bench29nov07gmt1602.asc'; %best background scatter file % metadata.factoryFile = 'factory_zsc_1172.asc'; % best background scatter file % metadata.ringFile = 'ringarea_1172.asc'; % best background scatter file % % these are [slope, offset] % metadata.cal.laserpower = [0.00081556 0]; % raw count to mW % metadata.cal.battery = [0.01 0]; % raw count to volts % metadata.cal.external = [0 0]; % none for LISST-ST % metadata.cal.laserreference = [0.00076741 0]; %raw count to mW % metadata.cal.pressure = [0.01 -10]; %raw count to meters % metadata.cal.temperature = [0.01 0]; % raw count to deg C % metadata.cal.volumeconversion = 45733; %in instrumentdata.txt % % this height will be tagged to all variables % metadata.inst_height = 0.31; % m pressure port height, about the same as the optics % metadata.STtopopening_height = 0; % m % metadata.STnsettlingsamples = 0; % a settling experiment % metadata.MOORING = '8482'; % % metadata.metafile = mfilename('fullpath'); % metadata.metafile_date = datestr(now); % metadata.metafile_version = '0.0'; % metadata.metafile_author = 'MM'; % % % if the following fields are missing, UNKOWN will be used % metadata.DATA_ORIGIN = gatt.DATA_ORIGIN; % with collaborator's data, could be USC, etc. % metadata.EXPERIMENT = gatt.EXPERIMENT; % metadata.PROJECT = gatt.PROJECT; % might also use OFA funding agency, such as MWRA, EPA, WCMG % metadata.DESCRIPTION = gatt.DESCRIPTION; % metadata.SciPi = gatt.SciPi; % metadata.Conventions = gatt.Conventions; % metadata.DATA_CMNT = ''; % your metadata % metadata.longitude = gatt.longitude; % negative degrees in western hemisphere % metadata.lonUnits = 'degrees_east'; % metadata.latitude = gatt.latitude; % always positive degrees in northern % metadata.latUnits = 'degrees_north'; % % metadata.dataFile = LISSTroot; % % metadata.goodsamples = [681 20528]; % use [1 Inf] for all data % % lisst = a structure of the data % tm = matlab time % tj = julian day % logFile = raw file name % rintensity = intensity on the detectors % laserxmit = laser transmission % batt = battery voltage % aux1 = auxiliary input % laserref = laser reference % press = pressure % temp = temperature % sizes = sizeclasses % sizeunits = units % logFile: source file name % zscData: background scatter data % % Settling experiments will be treated as simple time series. % Warning: this code will only fine the first year end rollover, if your % experiment is more than two years long, modify the code % 7/22/08 More metadata stuff % Written by Marinna Martini 2/18/06 function lisst = LISST100Xlog2cdf(LISSTroot, ncroot, metadata) % get the current SVN version- the value is automatically obtained in svn % is the file's svn.keywords which is set to "Revision" rev_info = 'SVN $Revision$'; disp(sprintf('%s %s running',mfilename,rev_info)) lisst = []; metadata.settling = 0; ndatacols = 40; disp(sprintf('%s now running %s',mfilename,datestr(now,0))) if ~exist('metadata','var'), disp('Metadata is required for processing') eval(['help ',mfilename]) end if ~exist('LISSTroot','var'), LISSTroot = ''; end [pathname, fname] = fileparts(LISSTroot); rawFile = fullfile(pathname,[fname,'.log']); % get the data file if fname(1) == 'S', disp('This is a settling experiment') metadata.settling = 1; if ~isfield(metadata,'STnsettlingsamples') buf = inputdlg('Enter the number of samples per settling experiment: ',... 'Required information',1,{'82'}); if isempty(buf), return, end metadata.STnsettlingsamples = str2double(buf{1}); end else metadata.STnsettlingsamples = 0; end if ~exist(rawFile,'file'), FilterSpec(1,:) = {'*.log','*.log raw LISST data file'}; [fname, pathname] = uigetfile(FilterSpec, 'Pick a LISST data file'); if isequal(fname,0) || isequal(pathname,0) disp('No raw LISST data file selcted by user') return end rawFile = fullfile(pathname, fname); end % metadata.zscFile = background scatter file chosen by user based on field conditions if ~isfield(metadata, 'zscFile') FilterSpec(1,:) = {'*.asc','*.asc LISST background scatter file'}; [fname, pathname] = uigetfile(FilterSpec, 'Pick a LISST background scatter file'); metadata.zscFile = fullfile(pathname, fname); end metadata.zscData = load(metadata.zscFile); if isempty(metadata.zscData) disp('No background scatter file data available, aborting') return end % metadata.factoryFile = factory background scatter file if ~isfield(metadata, 'factoryFile') FilterSpec(1,:) = {'*.asc','*.asc LISST factory background scatter file'}; [fname, pathname] = uigetfile(FilterSpec, 'Pick a LISST factory background scatter file'); metadata.factoryFile = fullfile(pathname, fname); end metadata.factoryData = load(metadata.factoryFile); if isempty(metadata.factoryData) disp('No factory background scatter file data available, aborting') return end % metadata.ringFile = ring area calibration file if ~isfield(metadata, 'ringFile') FilterSpec(1,:) = {'*.asc','*.asc LISST ring file'}; [fname, pathname] = uigetfile(FilterSpec, 'Pick a LISST ring file'); metadata.ringFile = fullfile(pathname, fname); end metadata.ringData = load(metadata.ringFile); if isempty(metadata.ringData) disp('No ring file data available, aborting') return end % figure out the time if ~isfield(metadata,'deployment_start'), % guess from the data file time stamp junk = dir(rawFile); dvec = datevec(datenum(junk.date)); disp(sprintf('Deployment year %d taken from data file time stamp',dvec(1))) disp(sprintf('Deployment month %d taken from data file time stamp',dvec(2))) else % take it from the in water time dvec = datevec(datenum(metadata.deployment_start)); disp(sprintf('Deployment year %d taken from in water time',dvec(1))) disp(sprintf('Deployment month %d taken from in water time',dvec(2))) end if isfield(metadata,'goodsamples'), goodsamples = metadata.goodsamples; if goodsamples(1) < 1, goodsamples(1) = 1; end %if goodsamples(2) > nsamples, goodsamples(2) = nsamples; end %data = data(goodsamples(1):goodsamples(2),:); end if isfield(metadata, 'type'), metadata.type = lower(metadata.type); else metadata.type = {'A','B','C','F'}; [n,v] = listdlg('PromptString','Select type of instrument:',... 'SelectionMode','single','ListString',metadata.type); if ~v, disp('LISST type is required information') return end metadata.type = lower(metadata.type{n}); end disp(sprintf('The LISST type is %s',upper(metadata.type))) if ~isfield(metadata, 'serial'), metadata.serial = ''; disp('No serial number provided') end disp('defining netCDF') pathname = fileparts(rawFile); cdfFile = fullfile(pathname,[ncroot,'.cdf']); cdf = netcdf(cdfFile,'clobber'); definenc(cdf, metadata); fid = fopen(rawFile,'r'); tic; count = ndatacols; cdfidx = 0; dataidx = 0; while count == ndatacols, dataidx = dataidx + 1; [data,count] = fscanf(fid, '%f', ndatacols); if isempty(data), break; end data = data'; nzeros = length(find(data == 0)); if metadata.settling && (nzeros == count-1), % this is a header, do nothing disp(sprintf('skipping record %d as settling experiment header',dataidx)) elseif dataidx >= goodsamples(1), cdfidx = cdfidx + 1; rec(cdfidx) = dataidx; % compute the time stamps % yearday includes the first day, so dayone is decremented by one dayone = datenum([dvec(1),1,1,0,0,0])-1; % the first index of yearday is Jan 1 of the given year yearday = floor(data(:,39)./100); hour = rem(data(:,39),100); min = floor(data(:,40)./100); sec = rem(data(:,40),100); tm(cdfidx) = dayone+yearday+hour./24+min./(24*60)+sec./(24*3600); tj(cdfidx) = julian(datevec(tm(cdfidx))); % save yday for later yday(cdfidx) = yearday; % parse the other data rintensity(cdfidx,:) = data(:,1:32); laserxmit(cdfidx) = data(:,33); batt(cdfidx) = data(:,34); aux1(cdfidx) = data(:,35); laserref(cdfidx) = data(:,36); press(cdfidx) = data(:,37); temp(cdfidx) = data(:,38); cdf{'rec'}(cdfidx,1,1,1) = rec(cdfidx); cdf{'time'}(cdfidx,1,1,1) = floor(tj(cdfidx)); cdf{'time2'}(cdfidx,1,1,1) = (tj(cdfidx) - floor(tj(cdfidx))).*(3600*24*1000); cdf{'intensity'}(cdfidx,:,1,1) = rintensity(cdfidx,:); cdf{'ltrans'}(cdfidx,1,1,1) = laserxmit(cdfidx); cdf{'batt'}(cdfidx,1,1,1) = batt(cdfidx); cdf{'lref'}(cdfidx,1,1,1) = laserref(cdfidx); cdf{'press'}(cdfidx,1,1,1) = press(cdfidx); cdf{'temp'}(cdfidx,1,1,1) = temp(cdfidx); cdf{'aux1'}(cdfidx,1,1,1) = aux1(cdfidx); end if dataidx > goodsamples(2), break; end if exist('tm','var'), if cdfidx == 1, disp(sprintf('First date stamp is %s',datestr(datenum(gregorian(tj(1)))))); end if cdfidx<50 && ~rem(cdfidx,10), disp(sprintf('Finished sample #%d %s %4.2f min elapsed',cdfidx,datestr(tm(cdfidx)),toc./60)), end if cdfidx>50 && ~rem(cdfidx,100), disp(sprintf('Finished sample #%d %s %4.2f min elapsed',cdfidx,datestr(tm(cdfidx)),toc./60)), end end end metadata.nsamples = cdfidx; disp(sprintf('%d samples stored to cdf file',cdfidx)) fclose(fid) % must correct time if there was a year rollover if (yday(1) > 1), % we didn't start at the beginning of a year yendidx = find(yday == 365,1,'last'); if ~isempty(yendidx), % a new year started during our experiment disp(sprintf('Found a year rollover at index %d, correcting for it',yendidx)) tj(yendidx+1:end) = tj(yendidx+1:end)+365; % TODO fix this leap year code % tg = gregorian(tj); % if (any(~mod(tg(:,1),4)) && any(mod(tg(:,1),100))) || any(~mod(tg(:,1),400)), % then leap % this next line bombs for starts after feb! % mar1idx = find(tg(:,2) > 2 ,1,'first'); % if ~isempty(mar1idx) % disp(sprintf('Found a leap year Mar 1 at index %d, correcting for it',mar1idx)) % tj(mar1idx-1:end) = tj(mar1idx-1:end)+1; % end % end cdf{'time'}(:,1,1,1) = floor(tj); end end cdf.history = ncchar(sprintf('Converted by %s %s;\n',mfilename,rev_info)); cdf.DELTA_T = gmean(diff(tj)).*(24*3600); cdf.start_time = ncchar(datestr(datenum(gregorian(tj(1))))); cdf.stop_time = ncchar(datestr(datenum(gregorian(tj(end))))); cdf.original_data_file = ncchar(LISSTroot); add_minmaxvalues(cdf) add_fillvalues(cdf,1E35) % add depth attributes to variables % uses the height of the pressure port or whatever is given by inst_height add_depth_atts(cdf) close(cdf) lisst.tm = tm; lisst.tj = tj; lisst.logFile = rawFile; %clear yearday hour min sec dayone % parse the other data lisst.rintensity = rintensity; lisst.laserxmit = laserxmit; lisst.batt = batt; lisst.aux1 = aux1; lisst.laserref = laserref; lisst.press = press; lisst.temp = temp; lisst.sizes = sizeclass(metadata.type); if metadata.type == 'a' || metadata.type == 'f', lisst.sizeunits = 'ring number'; else lisst.sizeunits = 'degrees'; end lisst.zscData = load(metadata.zscFile); return function definenc(cdf, metadata) % data from read_globalatts if isfield(metadata,'DATA_ORIGIN'), cdf.DATA_ORIGIN = ncchar(metadata.DATA_ORIGIN); end if isfield(metadata,'EXPERIMENT'), cdf.EXPERIMENT = ncchar(metadata.EXPERIMENT); end if isfield(metadata,'PROJECT'), cdf.PROJECT = ncchar(metadata.PROJECT); end if isfield(metadata,'DESCRIPTION'), cdf.DESCRIPTION = ncchar(metadata.DESCRIPTION); end if isfield(metadata,'SciPi'), cdf.SciPi = ncchar(metadata.SciPi); end if isfield(metadata,'Conventions'), cdf.Conventions = ncchar(metadata.Conventions); end if isfield(metadata,'DATA_CMNT'), cdf.DATA_CMNT = ncchar(metadata.DATA_CMNT); end if isfield(metadata,'latitude'), cdf.latitude = metadata.latitude; else cdf.latitude = ncfloat(999); end if isfield(metadata,'latUnits'), cdf.latUnits = metadata.latUnits; end if isfield(metadata,'longitude'), cdf.longitude = ncfloat(metadata.longitude); else cdf.longitude = ncfloat(999); end if isfield(metadata,'lonUnits'), cdf.lonUnits = metadata.lonUnits; end if isfield(metadata,'deployment_start'), cdf.deployment_start = ncfloat(metadata.deployment_start); else cdf.Deployment_date = ncfloat(999); end if isfield(metadata,'deployment_end'), cdf.deployment_end = ncfloat(metadata.deployment_end); else cdf.Recovery_date = ncfloat(999); end if isfield(metadata,'DATA_SUBTYPE'), cdf.DATA_SUBTYPE = ncchar(metadata.DATA_SUBTYPE); end if isfield(metadata,'MOORING'), cdf.MOORING = ncchar(metadata.MOORING); end if isfield(metadata,'WATER_DEPTH'), cdf.WATER_DEPTH = ncfloat(metadata.WATER_DEPTH); end cdf.CREATION_DATE = ncchar(datestr(now)); cdf.DATA_TYPE = ncchar('TIME'); cdf.COORD_SYSTEM = ncchar('GEOGRAPHICAL'); cdf.WATER_MASS = ncchar('?'); cdf.COMPOSITE = nclong(0); cdf.POS_CONST = nclong(0); cdf.DEPTH_CONST = nclong(1); % depth will change cdf.FILL_FLAG = nclong(1); cdf.VAR_FILL = ncfloat(1e35); cdf.water_depth = ncfloat(metadata.WATER_DEPTH); cdf.INST_TYPE = ncchar('Sequoia LISST100X'); cdf.inst_height = ncfloat(metadata.inst_height); cdf.inst_depth = ncfloat(metadata.WATER_DEPTH-metadata.inst_height); if metadata.settling, cdf.STnsettlingsamples = metadata.STnsettlingsamples; cdf.STtopopening_height = metadata.STtopopening_height; end cdf.ringFile = ncchar(metadata.ringFile); cdf.ringData = ncdouble(metadata.ringData); cdf.factoryFile = ncchar(metadata.factoryFile); cdf.factoryData = ncdouble(metadata.factoryData); cdf.zscFile = ncchar(metadata.zscFile); cdf.zscData = ncdouble(metadata.zscData); cdf.cal_laserpower = metadata.cal.laserpower; % raw count to mW cdf.cal_battery = metadata.cal.battery; % raw count to volts cdf.cal_external = metadata.cal.external; % none for LISST-ST cdf.cal_laserreference = metadata.cal.laserreference; %raw count to mW cdf.cal_pressure = metadata.cal.pressure; %raw count to meters cdf.cal_temperature = metadata.cal.temperature; % raw count to deg C cdf.cal_volumeconversion = metadata.cal.volumeconversion; %in instrumentdata.txt %% Dimensions: cdf('time') = 0; cdf('depth') = 1; cdf('lat') = 1; cdf('lon') = 1; cdf('rings') = 32; %% Variables and attributes: %% coordinate variables: cdf{'time'} = nclong('time') ; cdf{'time'}.FORTRAN_format = ncchar('F10.2'); cdf{'time'}.units = ncchar('True Julian Day'); cdf{'time'}.type = ncchar('EVEN'); cdf{'time'}.epic_code = nclong(624); cdf{'time2'} = nclong('time') ; cdf{'time2'}.FORTRAN_format = ncchar('F10.2'); cdf{'time2'}.units = ncchar('msec since 0:00 GMT'); cdf{'time2'}.type = ncchar('EVEN'); cdf{'time2'}.epic_code = nclong(624); cdf{'depth'} = ncfloat('depth') ; cdf{'depth'}.FORTRAN_format = ncchar('F10.2'); cdf{'depth'}.units = ncchar('m'); cdf{'depth'}.type = ncchar('EVEN'); cdf{'depth'}.epic_code = nclong(3); cdf{'lat'} = ncfloat('lat') ; cdf{'lat'}.FORTRAN_format = ncchar('F10.2'); cdf{'lat'}.units = ncchar('degrees_north'); cdf{'lat'}.type = ncchar('EVEN'); cdf{'lat'}.epic_code = nclong(500); cdf{'lon'} = ncfloat('lon') ; cdf{'lon'}.FORTRAN_format = ncchar('F10.2'); cdf{'lon'}.units = ncchar('degrees_east'); cdf{'lon'}.type = ncchar('EVEN'); cdf{'lon'}.epic_code = nclong(502); cdf{'rings'} = ncfloat('rings') ; cdfobj = cdf{'rings'}; cdfobj.FORTRAN_format = ncchar('F10.2'); cdfobj(:) = sizeclass(metadata.type); if metadata.type == 'a' || metadata.type == 'f', cdfobj.units = 'ring number'; else cdfobj.units = 'degrees'; end %% record variables: cdf{'rec'} = ncdouble('time', 'depth', 'lat', 'lon'); cdfobj = cdf{'rec'}; cdfobj.long_name = ncchar('Record Number'); cdfobj.units = ncchar('count'); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.valid_range = ncfloat([1 2^32]); cdf{'intensity'} = ncdouble('time', 'rings', 'lat', 'lon'); cdfobj = cdf{'intensity'}; cdfobj.long_name = ncchar('DETECTOR LIGHT INTENSITY '); cdfobj.units = ncchar(''); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(metadata.serial); cdfobj.valid_range = ncfloat([0 255]); cdf{'ltrans'} = ncfloat('time', 'depth','lat','lon'); cdfobj = cdf{'ltrans'}; cdfobj.long_name = ncchar('LASER TRANSMISSION '); cdfobj.units = ncchar('unknown'); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(metadata.serial); cdfobj.valid_range = ncfloat([0 10000]); cdf{'batt'} = ncfloat('time', 'depth','lat','lon'); cdfobj = cdf{'batt'}; cdfobj.long_name = ncchar('BATTERY VOLTAGE '); cdfobj.units = ncchar('counts'); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(metadata.serial); cdfobj.valid_range = ncfloat([0 10000]); cdf{'aux1'} = ncfloat('time', 'depth','lat','lon'); cdfobj = cdf{'aux1'}; cdfobj.long_name = ncchar('AXILIARY INPUT ! '); cdfobj.units = ncchar('counts'); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(metadata.serial); cdfobj.valid_range = ncfloat([0 4096]); cdf{'lref'} = ncfloat('time', 'depth','lat','lon'); cdfobj = cdf{'lref'}; cdfobj.long_name = ncchar('LASER REFERENCE'); cdfobj.units = ncchar('counts'); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(metadata.serial); cdfobj.valid_range = ncfloat([0 10000]); cdf{'press'} = ncfloat('time', 'depth', 'lat', 'lon'); cdfobj = cdf{'press'}; cdfobj.long_name = ncchar('RAW PRESSURE '); cdfobj.epic_code = nclong(1); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(' '); cdfobj.valid_range = ncfloat([0 10000]); cdf{'temp'} = ncfloat('time', 'depth', 'lat', 'lon') ; cdfobj = cdf{'temp'}; cdfobj.long_name = ncchar('RAW TEMPERATURE '); cdfobj.units = ncchar('counts'); cdfobj.sensor_depth = ncfloat(0); cdfobj.minimum = ncfloat(0); cdfobj.maximum = ncfloat(0); cdfobj.serial_number = ncchar(' '); cdfobj.valid_range = ncfloat([-50 50]); return function sc = sizeclass(type) switch type, case 'a', sc = 1:32; case 'b', sc = [0.106 0.125 0.148 0.174 0.206 0.243 0.287 0.338 ... 0.40 0.47 0.56 0.66 0.77 0.91 1.08 1.27 ... 1.50 1.77 2.09 2.46 2.91 3.43 4.05 4.78 ... 5.64 6.65 7.85 9.26 10.93 12.90 15.22 17.96]; case 'c', sc = [0.053 0.063 0.074 0.087 0.103 0.121 0.143 0.169 ... 0.20 0.24 0.28 0.33 0.39 0.46 0.54 0.64 ... 0.75 0.89 1.04 1.23 1.45 1.72 2.02 2.39 ... 2.82 3.33 3.93 4.63 5.47 6.45 7.61 8.98]; case 'f', sc = 1:32; end return % add depth attributes to variables % uses the height of the pressure port or whatever is given by inst_height function add_depth_atts(cdf) theVars = var(cdf); for i = 1:length(theVars), if ~strcmp(ncnames(theVars{i}),'time') && ... ~strcmp(ncnames(theVars{i}),'time2') && ... ~strcmp(ncnames(theVars{i}),'lat') && ... ~strcmp(ncnames(theVars{i}),'lon') && ... ~strcmp(ncnames(theVars{i}),'rings'), theVars{i}.sensor_depth = ncfloat(cdf.WATER_DEPTH(:)-cdf.inst_height(:)); theVars{i}.sensor_height = ncfloat(cdf.inst_height(:)); end end return