function [PencilData, PencilHeader, PencilSwitches] = readpencil(fname, verbose) % READPENCIL - read in data from Imagenex pencil beam sonar % [PencilData, PencilHeader, PencilSwitches] = readpencil(fname, verbose); % % fname = file name % % PencilData.imagedata: [ping x bin double], image data with header removed % % HeaderData is a dump of the header info sent with *each ping* % HeaderData.ReturnDataHeaderType = IGX/IMX etc. % HeaderData.HeadID = ID number % HeaderData.HeadPosition = head position in counts % HeaderData.HeadAngle = head rotation angle in degrees % HeaderData.StepDirection = direction of rotation, -1 = CCW, 1 = CW % HeaderData.Range = range setting % HeaderData.ProfileRange = first digitized value above the threshold in cm % HeaderData.NDataBytes = number of byte in the ping % HeaderData.NReturnBytes = number of bytes in the ping + header % HeaderData.StepSize = degrees per step of rotation % HeaderData.FileName = name of the file read % HeaderData.NPoints = number of data points returned for the ping % HeaderData.SectorWidth = size of sector swept in degrees % HeaderData.StartGain = gain setting word from switches (for .81a files) % % PencilSwitches is a dump of the switches sent to the sonar for this sweep % PencilSwitches = % HeadID: 16 % Range: 2 % Reserved: [0 0 0 0 0 0] % RevHold: 'Reverse Step Direction' % MasterSlave: 'Send Data Slave Mode' % StartGain: 5 % LOGF: 20 % Absorption: 0.0600 % TrainAngle: 0 % SectorWidth: 150 % StepSize: 0.3000 % PulseLength: 10 % DataPoints: 500 % DataBits: 8 % UpBaud: 6 % Profile: 'OFF' % Calibrate: 'Calibrate' % SwitchDelay: 2 % Frequency: 900 % 16jul07 fix bad bit parsing of Profile Range and NDataBytes in header % Written by Marinna Martini 5/22/07 % for the U.S. Geological Survey, WOods Hole Field Center % % 07/06/07 ETM- made all StepSize == 0.3 pre conv. with Doug Wilson % and made a case to exit cleanly if all header words are 0 NRETURNHEADERBYTES = 12; NWHSCFILEHEADERBYTES = 61; N81AFILEHEADERBYTES = 100; PencilData = []; PencilHeader = []; PencilSwitches = []; if ~exist('fname', 'var') || ~exist(fname, 'file'), [fname, pathname] = uigetfile( ... {'*.P*','WHSC logger files (S1*.P*)'; ... '*.81A','Win8812W output files (*.81A)'; ... '*.*', 'All Files (*.*)'}); if isempty(fname), return, end fname = fullfile(pathname, fname); end fid = fopen(fname,'r'); if fid == -1, disp(['error reading file ',fname]); return else disp(['Now reading file ', fname]); end PencilHeader.FileName = fname; if ~exist('verbose','var'), verbose = 1; end % detect which file we have % read the first few bytes tag = fread(fid,3,'uchar'); fseek(fid,0,'bof'); switch(char(tag)') case 'The' % this is USGS WHSC logger data % extract year info from file d = dir(fname); datayear = datevec(d.datenum); % first part is the file name, as written by the logger, % which is essentially the time stamp fheader = fread(fid,NWHSCFILEHEADERBYTES,'uchar'); [PencilTime, PencilSwitches] = parseLoggerHeader(fheader, datayear(1)); clear d % now at the data % decipher the first 13 bytes of sonar header so we know what we are reading sheader = fread(fid,NRETURNHEADERBYTES,'uchar'); header = parsePencilheader(sheader, PencilSwitches, verbose); % display them once if isempty(header) %handles the empty header case return end disp(sprintf('Number of return bytes per ping = %d',header.NReturnBytes)); npoints = header.NReturnBytes-NRETURNHEADERBYTES+1; disp(sprintf('Number of points per ping = %d',npoints)); PencilHeader.PencilTime = PencilTime; % pass out the time stamp clear sheader % now figure out how many pings of data are in this file from the file size and header info. fileinfo = dir(fname); disp(fileinfo.bytes); npings = (fileinfo.bytes-NWHSCFILEHEADERBYTES)./header.NReturnBytes; disp(sprintf('Number of pings in the file = %d',npings)) fseek(fid, NWHSCFILEHEADERBYTES, 'bof'); % go to data start raw = fread(fid,'uchar'); % read in all the ping data endval=header.NReturnBytes*floor(npings); imagedata = reshape(raw(1:endval), header.NReturnBytes, floor(npings)); % let's check out the header for each ping for iping = 1:npings, header = parsePencilheader(imagedata(1:13,iping), PencilSwitches, 0); % rearrange the output for convenience PencilHeader.ReturnDataHeaderType{iping} = header.ReturnDataHeaderType; PencilHeader.HeadID(iping) = header.HeadID; PencilHeader.HeadPosition(iping) = header.HeadPosition; PencilHeader.HeadAngle(iping) = header.HeadAngle; PencilHeader.StepDirection(iping) = header.StepDirection; PencilHeader.Range(iping) = header.Range; PencilHeader.ProfileRange(iping) = header.ProfileRange; PencilHeader.NDataBytes(iping) = header.NDataBytes; PencilHeader.NReturnBytes(iping) = header.NReturnBytes; end PencilData.imagedata = imagedata(14:end,:); % pass back just data PencilHeader.NPoints = PencilSwitches.DataPoints; PencilHeader.StepSize = PencilSwitches.StepSize; PencilHeader.SectorWidth = PencilSwitches.SectorWidth; case '81A' % this is data from an 81A file fheader = fread(fid,N81AFILEHEADERBYTES,'uchar'); [PencilTime, PencilSwitches] = parse81AHeader(fheader,1); % now at the data % decipher the first 13 bytes of sonar header so we know what we are reading sheader = fread(fid,NRETURNHEADERBYTES,'uchar'); header = parsePencilheader(sheader, PencilSwitches, verbose); % display them once PencilHeader.FileName = fname; PencilHeader.NPoints = header.NDataBytes; PencilHeader.StepSize = PencilSwitches.StepSize; PencilHeader.SectorWidth = PencilSwitches.SectorSize; clear sheader % now figure out how many pings of data are in this file from the file size and header info. fileinfo = dir(fname); disp(sprintf('The file size is %d',fileinfo.bytes)); npings = (fileinfo.bytes)./(PencilSwitches.TotalBytes); disp(sprintf('Number of pings in the file = %d',npings)) % allocate memory fseek(fid, 0, 'bof'); % go to data start raw = fread(fid,'uchar'); % read in all the ping data at once imagedata = reshape(raw, PencilSwitches.TotalBytes, npings); PencilData.raw = imagedata; % let's check out the header for each ping for iping = 1:npings, [PencilTime, switches] = parse81AHeader(imagedata(1:100,iping),0); PencilHeader.PencilTime(iping) = PencilTime; header = parsePencilheader(imagedata(101:112,iping), PencilSwitches, 0); if ~isempty(header), % rearrange the output for convenience PencilHeader.ReturnDataHeaderType{iping} = header.ReturnDataHeaderType; PencilHeader.HeadID(iping) = header.HeadID; PencilHeader.HeadPosition(iping) = header.HeadPosition; PencilHeader.HeadAngle(iping) = header.HeadAngle; PencilHeader.StepDirection(iping) = header.StepDirection; PencilHeader.Range(iping) = header.Range; PencilHeader.ProfileRange(iping) = header.ProfileRange; PencilHeader.NDataBytes(iping) = header.NDataBytes; % this only works for .81a files- % logger generated .P## and .R## files only do switches once PencilHeader.StartGain(iping) = switches.StartGain; if ~isempty(header.NReturnBytes), PencilHeader.NReturnBytes(iping) = header.NReturnBytes; else PencilHeader.NReturnBytes(iping) = NaN; disp(sprintf('Problem at ping #%d',iping)) end else disp(sprintf('Problem at ping #%d',iping)) end end PencilData.imagedata = imagedata(113:end,:); % pass back just data case {'IGX','IMX','IPX'} % .81a is also an IMX, so was read twice % case {'IGX','IMX','IPX'} % this is USGS WHSC logger *.tmp format % extract year info from file d = dir(fname); PencilTime = julian(datevec(d.datenum)); PencilHeader.PencilTime = PencilTime; % pass out the time stamp % need to know the data bits, 4, 8 or 16! button = questdlg('Select number of Data Bits','No SwitchInformation','4','8','16','8'); PencilSwitches.DataBits = str2num(char(button)); % this file format has no header, it goes right to the data % decipher the first 13 bytes of sonar header so we know what we are reading sheader = fread(fid,NRETURNHEADERBYTES,'uchar'); header = parsePencilheader(sheader, PencilSwitches, verbose); % display them once disp(sprintf('Number of return bytes per ping = %d',header.NReturnBytes)); npoints = header.NReturnBytes-NRETURNHEADERBYTES+1; disp(sprintf('Number of points per ping = %d',npoints)); clear sheader % now figure out how many pings of data are in this file from the file size and header info. fileinfo = dir(fname); disp(fileinfo.bytes); npings = (fileinfo.bytes)./header.NReturnBytes; disp(sprintf('Number of pings in the file = %d',npings)) fseek(fid, 0, 'bof'); % go to data start raw = fread(fid,'uchar'); % read in all the ping data imagedata = reshape(raw, header.NReturnBytes, npings); % let's check out the header for each ping for iping = 1:npings, header = parsePencilheader(imagedata(1:13,iping), PencilSwitches, 0); % rearrange the output for convenience PencilHeader.ReturnDataHeaderType{iping} = header.ReturnDataHeaderType; PencilHeader.HeadID(iping) = header.HeadID; PencilHeader.HeadPosition(iping) = header.HeadPosition; PencilHeader.HeadAngle(iping) = header.HeadAngle; PencilHeader.StepDirection(iping) = header.StepDirection; PencilHeader.Range(iping) = header.Range; PencilHeader.ProfileRange(iping) = header.ProfileRange; PencilHeader.NDataBytes(iping) = header.NDataBytes; PencilHeader.NReturnBytes(iping) = header.NReturnBytes; end PencilData.imagedata = imagedata(14:end,:); % pass back just data PencilHeader.NPoints = header.NReturnBytes-NRETURNHEADERBYTES+1; %PencilHeader.StepSize = PencilSwitches.StepSize; %PencilHeader.SectorWidth = PencilSwitches.SectorWidth; otherwise disp('Data is unrecognized') return end fclose(fid); return PencilSwitches = parsePencilswitches(fheader(35:end)); % now at the data % decipher the first 13 bytes of sonar header so we know what we are reading sheader = fread(fid,NRETURNHEADERBYTES,'uchar'); PencilHeader = parsePencilheader(sheader, PencilSwitches, verbose); % display them once disp(sprintf('Number of return bytes per ping = %d',PencilHeader.NReturnBytes)); npoints = PencilHeader.NReturnBytes - NRETURNHEADERBYTES; disp(sprintf('Number of points per ping = %d',npoints)); PencilHeader.PencilTime = PencilTime; % pass out the time stamp clear sheader % now figure out how many pings of data are in this file from the file size and header info. fileinfo = dir(fname); disp(fileinfo.bytes); npings = (fileinfo.bytes-NFILEHEADERBYTES)./PencilHeader.NReturnBytes; disp(sprintf('Number of useable pings = %d',npings)) % Marinna reads the data fseek(fid, NFILEHEADERBYTES, 'bof'); % go to data start A1 = fread(fid,'uchar'); % read in all the ping data imagedata = reshape(A1, PencilHeader.NReturnBytes, npings); fclose(fid); PencilData.rawimage = imagedata; % pass back unadulterated data to play with % let's check out the header for each ping clear PencilHeader for iping = 1:npings, PencilHeader(iping) = parsePencilheader(imagedata(1:13,iping), PencilSwitches, 0); end return %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % read a WHSC logger file header from a S1*.F* file % function [PencilTime, PencilSwitches] = parseLoggerHeader(fheader, datayear) % bytes 1:33 are: % The file name is \S1\S1031414.p12 % it is possible that this can vary % extract the time information from the file header cfheader = char(fheader); MM = sscanf(cfheader(24:25),'%d'); dd = sscanf(cfheader(26:27),'%d'); hh = sscanf(cfheader(28:29),'%d'); mm = sscanf(cfheader(32:33),'%d'); PencilTime = julian([datayear, MM, dd, hh, mm, 0]); clear MM dd hh mm cfheader % next thing the logger writes out are the switches. 29 bytes. % not sure why byte #34 is written. It is ASCII LF = x0A % pull out switch data for indexing convenience PencilSwitches = parsePencilswitches(fheader(35:end)); return %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % read the header from an output file from live operations (win881A.exe) function [PencilTime, File81AHeader] = parse81AHeader(fheader, verbose) File81AHeader.ReturnDataHeaderType = char(fheader(1:3))'; junk = [0 128 250 500]; File81AHeader.NDataBytes = junk(fheader(4)+1); File81AHeader.TotalBytes = fheader(5).*256+fheader(6); File81AHeader.nToRead = fheader(7).*256+fheader(8); % date format 11-May-2007.21:30:05 dstr = char(fheader(9:29)'); dstr(12) = ' '; hsec = str2num(char(fheader(31:32))); % byte 30 is '.' and byte 33 is null PencilTime = julian(datevec(datenum(dstr))); File81AHeader.xBytes = fheader(35).*128; File81AHeader.TotalBytes = File81AHeader.TotalBytes + File81AHeader.xBytes; File81AHeader.StepDirection = bitget(fheader(38),8); % 1 = CW File81AHeader.Orientation = bitget(fheader(38),7); % 1 = Up junk = {'Sector','Polar','Sidescan'}; n = bitshift(bitand(fheader(38), 56),-3); File81AHeader.Mode = junk{n+1}; % junk = [0 0.3 0.6 0.9 1.2 2.4]; junk = [0.3 0.6 0.9 1.2 2.4]; % can't use the leading 0 File81AHeader.StepSize = junk(bitand(fheader(38), 7)+1); % StepSize is ALWAYS .3 for Pencil- Doug Wilson, Pers.Comm 6/07 % File81AHeader.StepSize = 0.3; File81AHeader.StartGain = fheader(39); File81AHeader.SectorSize = fheader(40).*3; % I don't think this should be -210 etm % File81AHeader.TrainAngle = (fheader(41).*3) - 210; File81AHeader.TrainAngle = (fheader(41).*3); File81AHeader.RangeOffset = fheader(42); File81AHeader.Absorption = fheader(43).*0.01; File81AHeader.ProfileGrid = bitget(fheader(44),8); % 1 = ON File81AHeader.Zero = bitget(fheader(44),7); % 1 = Dn junk = [4 8 14]; n = bitshift(bitand(fheader(44), 56),-3); File81AHeader.DataBits = junk(n+1); File81AHeader.LogF = bitand(fheader(44), 7).*10; File81AHeader.PulseLength = fheader(45).*10; junk = {'Off','PointsOnly','Lo-mix','Hi-Mix'}; File81AHeader.Profile = junk{fheader(46)+1}; if bitget(fheader(47),8), File81AHeader.SoundVelocity = (bitand(fheader(46),127).*256+fheader(47))/10; else File81AHeader.SoundVelocity = 1500; end File81AHeader.Comment = fheader(49:80); File81AHeader.OperatingFrequency = fheader(81).*256+fheader(82); if bitget(fheader(83),8), File81AHeader.AzimuthDrivePosition = (bitand(fheader(83),127).*256+fheader(84)); else File81AHeader.AzimuthDrivePosition = NaN; end if verbose, disp(File81AHeader) end return %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % read the header that comes with each ping. % applies to logger and live data function PencilHeader = parsePencilheader(sheader, switches, verbose) %check that the header has something in it (etm 7/6/07) xx=find(sheader==0); if (length(xx)==length(sheader)) display('header is all zeros- cannot process- exiting readpencil'); PencilHeader = []; return; end clear xx %now start parsing the header PencilHeader.ReturnDataHeaderType = char(sheader(1:3))'; PencilHeader.HeadID = sheader(4); b=dec2base(sheader(5),2,8); % get the bits if b(7), PencilHeader.SerialStatus = 'Switches Accepted'; end if b(8), PencilHeader.SerialStatus = 'Character Overrun'; end % head position % explicitly the method from the Imagenex manual Yuk! % HI = bitand(sheader(7),hex2dec('3E')); % HI = bitshift(HI,-1); % shift right % LO1 = bitand(sheader(7),hex2dec('01')); % LO1 = bitshift(LO1,7); % LO2 = bitand(sheader(6),hex2dec('7F')); % LO = bitor(LO1, LO2); % HI = bitshift(HI,8); % HP = bitor(HI, LO); % dec2bin(HP) % PencilHeader.HeadAngle = (HP-600)*0.3; % disp(sprintf('IMagenex method head angle %f', PencilHeader.HeadAngle)) % Doug Wilson's method is more concise. He probably didn't write the manual PencilHeader.HeadPosition = bitand(63,sheader(7)).*128 + bitand(127,sheader(6)); PencilHeader.HeadAngle = ( PencilHeader.HeadPosition-600).*0.3; % degrees %PencilHeader.HeadAngle = ( PencilHeader.HeadPosition-600).*switches.StepSize; % degrees if bitget(sheader(7),6), PencilHeader.StepDirection = 1; else PencilHeader.StepDirection = -1; end PencilHeader.Range = sheader(8); % meters % MM 10jul07 %PencilHeader.ProfileRange = bitand(127,sheader(10)).*127 + bitand(127,sheader(9)); % meters % bitshift(A, k) % negative = shift right PRHB = bitshift(bitand(sheader(10), 126), -1); PRLB = bitor(bitshift(bitand(sheader(10), 1), 7),bitand(sheader(9), 127)); PencilHeader.ProfileRange = bitor(bitshift(PRHB,8),PRLB); %PencilHeader.NDataBytes = bitand(127,sheader(12)).*127 + bitand(127,sheader(11)); % meters PRHB = bitshift(bitand(sheader(12), 126), -1); PRLB = bitor(bitshift(bitand(sheader(12), 1), 7),bitand(sheader(11), 127)); PencilHeader.NDataBytes = bitor(bitshift(PRHB,8),PRLB); % given the header info, figure out how many data point we're dealing with % the number of data bits is hardwired to 8 in the logger source code (new_bits) PencilHeader.NReturnBytes = []; PencilHeader.NZeroFillBytes = []; switch(PencilHeader.ReturnDataHeaderType), case {'IMX'}, % switch data command for data points = 25 if switches.DataBits == 4, % 128 Data Byes, 256 Points PencilHeader.NReturnBytes = 141; PencilHeader.NPoints = 256; elseif switches.DataBits == 8, % 252 Data Byes, 252 Points PencilHeader.NReturnBytes = 265; PencilHeader.NPoints = 252; PencilHeader.NZeroFillBytes = 19; elseif switches.DataBits == 16, % 500 Data Byes, 250 Points PencilHeader.NReturnBytes = 513; PencilHeader.NPoints = 250; end case {'IGX'}, % switch data command for data points = 50 if switches.DataBits == 4, PencilHeader.NReturnBytes = 265; % 252 Data Byes, 504 Points PencilHeader.NPoints = 504; elseif switches.DataBits == 8, PencilHeader.NReturnBytes = 513; % 500 Data Byes, 500 Points PencilHeader.NPoints = 500; PencilHeader.NZeroFillBytes = 27; else % 500 Data Bytes, 250 Points PencilHeader.NReturnBytes = 500; PencilHeader.NPoints = 250; end case {'IQX'}, % there is no IQX for the pencil beam sonar case {'IPX'}, % Profile = ON PencilHeader.NReturnBytes = 13; % 0 Data Byes, 0 Points PencilHeader.NZeroFillBytes = 15; PencilHeader.NPoints = 0; otherwise sprintf([PencilHeader.ReturnDataHeaderType,' not recognized']) return end if verbose, disp(PencilHeader) end return %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % this function is only used for logger acquired files- not .8a1's % read the settings sent to the sonar head % these are saved at the beginning of each data file in the logger function switches = parsePencilswitches(switchdata) % first two bytes are switch data header ID = 0xFE44 if (switchdata(1) ~= 254) || (switchdata(2) ~= 68), disp('Switch data header ID not found') end switches.HeadID = switchdata(3); switches.Range = switchdata(4); % 5 to 255 m in 5 m increments switches.Reserved(1) = switchdata(5); % always 0 b=dec2base(switchdata(6),2,8); % get the bits if b(1), switches.RevHold = 'Hold Head'; else switches.RevHold = 'Resume'; end if b(7), switches.RevHold = 'Reverse Step Direction'; else switches.RevHold = 'Normal Operation'; end b=dec2base(switchdata(7),2,8); % get the bits % I'm not too sure how to interpret the manual on this one % but this should not matter to logger data if b(1), if b(7), switches.MasterSlave = 'Transmit Slave Mode'; else switches.MasterSlave = 'Transmit Master Mode'; end end if b(2), if b(7), switches.MasterSlave = 'Send Data Slave Mode'; else switches.MasterSlave = 'Send Data Master Mode'; end end switches.Reserved(2) = switchdata(8); % always 0 switches.StartGain = switchdata(9); % 0 to 40dB in 1 dB increments switches.LOGF = switchdata(10).*10; % in dB switches.Absorption = switchdata(11)./100; % dB/m if switches.Absorption == 253, disp('Absorption value of 253 detected, this is not allowed') end % the .3 is ALWAYS StepSize and switches.TrainAngle = (switchdata(12).*3)-180; % degrees switches.SectorWidth = switchdata(13).*3; % degrees % pencil beam step increments are usually 0.3 degrees switches.StepSize = switchdata(14).*0.3; % degrees per step up to 2.4 if switches.StepSize > 2.4, disp('Step size greater than 2.4 detected, this is not possible') end switches.PulseLength = switchdata(15).*10; % microseconds switches.ProfileMinRange = switchdata(16)./10; % meters switches.Reserved(3:5) = switchdata(17:19); % always 0 switches.DataPoints = switchdata(20).*10; % 250 for IMX and 500 for IGX % resolution of data as data width... % 4 bits = 2 data points per byte % 8 bits = 1 data point per byte % 16 bits = 2 bytes per data point switches.DataBits = switchdata(21); % in bits switches.UpBaud = switchdata(22); % not yet used if switchdata(23), switches.Profile = 'ON'; else switches.Profile = 'OFF'; % return data will have IPX header end if switchdata(24), switches.Calibrate = 'Normal Operation'; else switches.Calibrate = 'Calibrate'; % head is moved to 0 degrees end % the delay to give a control program time to serially upload data switches.SwitchDelay = switchdata(25).*2; % milliseconds if switches.SwitchDelay == 253, disp('Switch Delay value of 253 detected, this is not allowed') end switches.Reserved(7) = NaN; switches.Frequency = ((switchdata(26)-100).*5)+675; % kHz % termination byte if (switchdata(27) ~= 253), disp('Switch data termination byte not found') end disp(switches) return