From c5175b20d26ebe996a4efb2c24cb564fb94b69a1 Mon Sep 17 00:00:00 2001 From: sawinglogz <3787776-sawinglogz@users.noreply.gitlab.com> Date: Mon, 31 May 2021 21:31:34 -0400 Subject: [PATCH] Move PRS1 file parsing into separate parser file. No change in functionality. Use git blame dd9a087 to follow the history before this refactoring. --- oscar/SleepLib/loader_plugins/prs1_loader.cpp | 478 ----------------- oscar/SleepLib/loader_plugins/prs1_parser.cpp | 490 +++++++++++++++++- 2 files changed, 488 insertions(+), 480 deletions(-) diff --git a/oscar/SleepLib/loader_plugins/prs1_loader.cpp b/oscar/SleepLib/loader_plugins/prs1_loader.cpp index e0bbd5da..68d50c05 100644 --- a/oscar/SleepLib/loader_plugins/prs1_loader.cpp +++ b/oscar/SleepLib/loader_plugins/prs1_loader.cpp @@ -43,152 +43,6 @@ // that change loader behaviour or modify channels. //******************************************************************************************** - -// CRC-16/KERMIT, polynomial: 0x11021, bit reverse algorithm -// Table generated by crcmod (crc-kermit) - -typedef quint16 crc16_t; -static crc16_t CRC16(unsigned char * data, size_t data_len, crc16_t crc=0) -{ - static const crc16_t table[256] = { - 0x0000U, 0x1189U, 0x2312U, 0x329bU, 0x4624U, 0x57adU, 0x6536U, 0x74bfU, - 0x8c48U, 0x9dc1U, 0xaf5aU, 0xbed3U, 0xca6cU, 0xdbe5U, 0xe97eU, 0xf8f7U, - 0x1081U, 0x0108U, 0x3393U, 0x221aU, 0x56a5U, 0x472cU, 0x75b7U, 0x643eU, - 0x9cc9U, 0x8d40U, 0xbfdbU, 0xae52U, 0xdaedU, 0xcb64U, 0xf9ffU, 0xe876U, - 0x2102U, 0x308bU, 0x0210U, 0x1399U, 0x6726U, 0x76afU, 0x4434U, 0x55bdU, - 0xad4aU, 0xbcc3U, 0x8e58U, 0x9fd1U, 0xeb6eU, 0xfae7U, 0xc87cU, 0xd9f5U, - 0x3183U, 0x200aU, 0x1291U, 0x0318U, 0x77a7U, 0x662eU, 0x54b5U, 0x453cU, - 0xbdcbU, 0xac42U, 0x9ed9U, 0x8f50U, 0xfbefU, 0xea66U, 0xd8fdU, 0xc974U, - 0x4204U, 0x538dU, 0x6116U, 0x709fU, 0x0420U, 0x15a9U, 0x2732U, 0x36bbU, - 0xce4cU, 0xdfc5U, 0xed5eU, 0xfcd7U, 0x8868U, 0x99e1U, 0xab7aU, 0xbaf3U, - 0x5285U, 0x430cU, 0x7197U, 0x601eU, 0x14a1U, 0x0528U, 0x37b3U, 0x263aU, - 0xdecdU, 0xcf44U, 0xfddfU, 0xec56U, 0x98e9U, 0x8960U, 0xbbfbU, 0xaa72U, - 0x6306U, 0x728fU, 0x4014U, 0x519dU, 0x2522U, 0x34abU, 0x0630U, 0x17b9U, - 0xef4eU, 0xfec7U, 0xcc5cU, 0xddd5U, 0xa96aU, 0xb8e3U, 0x8a78U, 0x9bf1U, - 0x7387U, 0x620eU, 0x5095U, 0x411cU, 0x35a3U, 0x242aU, 0x16b1U, 0x0738U, - 0xffcfU, 0xee46U, 0xdcddU, 0xcd54U, 0xb9ebU, 0xa862U, 0x9af9U, 0x8b70U, - 0x8408U, 0x9581U, 0xa71aU, 0xb693U, 0xc22cU, 0xd3a5U, 0xe13eU, 0xf0b7U, - 0x0840U, 0x19c9U, 0x2b52U, 0x3adbU, 0x4e64U, 0x5fedU, 0x6d76U, 0x7cffU, - 0x9489U, 0x8500U, 0xb79bU, 0xa612U, 0xd2adU, 0xc324U, 0xf1bfU, 0xe036U, - 0x18c1U, 0x0948U, 0x3bd3U, 0x2a5aU, 0x5ee5U, 0x4f6cU, 0x7df7U, 0x6c7eU, - 0xa50aU, 0xb483U, 0x8618U, 0x9791U, 0xe32eU, 0xf2a7U, 0xc03cU, 0xd1b5U, - 0x2942U, 0x38cbU, 0x0a50U, 0x1bd9U, 0x6f66U, 0x7eefU, 0x4c74U, 0x5dfdU, - 0xb58bU, 0xa402U, 0x9699U, 0x8710U, 0xf3afU, 0xe226U, 0xd0bdU, 0xc134U, - 0x39c3U, 0x284aU, 0x1ad1U, 0x0b58U, 0x7fe7U, 0x6e6eU, 0x5cf5U, 0x4d7cU, - 0xc60cU, 0xd785U, 0xe51eU, 0xf497U, 0x8028U, 0x91a1U, 0xa33aU, 0xb2b3U, - 0x4a44U, 0x5bcdU, 0x6956U, 0x78dfU, 0x0c60U, 0x1de9U, 0x2f72U, 0x3efbU, - 0xd68dU, 0xc704U, 0xf59fU, 0xe416U, 0x90a9U, 0x8120U, 0xb3bbU, 0xa232U, - 0x5ac5U, 0x4b4cU, 0x79d7U, 0x685eU, 0x1ce1U, 0x0d68U, 0x3ff3U, 0x2e7aU, - 0xe70eU, 0xf687U, 0xc41cU, 0xd595U, 0xa12aU, 0xb0a3U, 0x8238U, 0x93b1U, - 0x6b46U, 0x7acfU, 0x4854U, 0x59ddU, 0x2d62U, 0x3cebU, 0x0e70U, 0x1ff9U, - 0xf78fU, 0xe606U, 0xd49dU, 0xc514U, 0xb1abU, 0xa022U, 0x92b9U, 0x8330U, - 0x7bc7U, 0x6a4eU, 0x58d5U, 0x495cU, 0x3de3U, 0x2c6aU, 0x1ef1U, 0x0f78U, - }; - - for (size_t i=0; i < data_len; i++) { - crc = table[(*data ^ (unsigned char)crc) & 0xFF] ^ (crc >> 8); - data++; - } - return crc; -} - - -// CRC-32/MPEG-2, polynomial: 0x104C11DB7 -// Table generated by crcmod (crc-32-mpeg) - -typedef quint32 crc32_t; -static crc32_t CRC32(const unsigned char *data, size_t data_len, crc32_t crc=0xffffffffU) -{ - static const crc32_t table[256] = { - 0x00000000U, 0x04c11db7U, 0x09823b6eU, 0x0d4326d9U, - 0x130476dcU, 0x17c56b6bU, 0x1a864db2U, 0x1e475005U, - 0x2608edb8U, 0x22c9f00fU, 0x2f8ad6d6U, 0x2b4bcb61U, - 0x350c9b64U, 0x31cd86d3U, 0x3c8ea00aU, 0x384fbdbdU, - 0x4c11db70U, 0x48d0c6c7U, 0x4593e01eU, 0x4152fda9U, - 0x5f15adacU, 0x5bd4b01bU, 0x569796c2U, 0x52568b75U, - 0x6a1936c8U, 0x6ed82b7fU, 0x639b0da6U, 0x675a1011U, - 0x791d4014U, 0x7ddc5da3U, 0x709f7b7aU, 0x745e66cdU, - 0x9823b6e0U, 0x9ce2ab57U, 0x91a18d8eU, 0x95609039U, - 0x8b27c03cU, 0x8fe6dd8bU, 0x82a5fb52U, 0x8664e6e5U, - 0xbe2b5b58U, 0xbaea46efU, 0xb7a96036U, 0xb3687d81U, - 0xad2f2d84U, 0xa9ee3033U, 0xa4ad16eaU, 0xa06c0b5dU, - 0xd4326d90U, 0xd0f37027U, 0xddb056feU, 0xd9714b49U, - 0xc7361b4cU, 0xc3f706fbU, 0xceb42022U, 0xca753d95U, - 0xf23a8028U, 0xf6fb9d9fU, 0xfbb8bb46U, 0xff79a6f1U, - 0xe13ef6f4U, 0xe5ffeb43U, 0xe8bccd9aU, 0xec7dd02dU, - 0x34867077U, 0x30476dc0U, 0x3d044b19U, 0x39c556aeU, - 0x278206abU, 0x23431b1cU, 0x2e003dc5U, 0x2ac12072U, - 0x128e9dcfU, 0x164f8078U, 0x1b0ca6a1U, 0x1fcdbb16U, - 0x018aeb13U, 0x054bf6a4U, 0x0808d07dU, 0x0cc9cdcaU, - 0x7897ab07U, 0x7c56b6b0U, 0x71159069U, 0x75d48ddeU, - 0x6b93dddbU, 0x6f52c06cU, 0x6211e6b5U, 0x66d0fb02U, - 0x5e9f46bfU, 0x5a5e5b08U, 0x571d7dd1U, 0x53dc6066U, - 0x4d9b3063U, 0x495a2dd4U, 0x44190b0dU, 0x40d816baU, - 0xaca5c697U, 0xa864db20U, 0xa527fdf9U, 0xa1e6e04eU, - 0xbfa1b04bU, 0xbb60adfcU, 0xb6238b25U, 0xb2e29692U, - 0x8aad2b2fU, 0x8e6c3698U, 0x832f1041U, 0x87ee0df6U, - 0x99a95df3U, 0x9d684044U, 0x902b669dU, 0x94ea7b2aU, - 0xe0b41de7U, 0xe4750050U, 0xe9362689U, 0xedf73b3eU, - 0xf3b06b3bU, 0xf771768cU, 0xfa325055U, 0xfef34de2U, - 0xc6bcf05fU, 0xc27dede8U, 0xcf3ecb31U, 0xcbffd686U, - 0xd5b88683U, 0xd1799b34U, 0xdc3abdedU, 0xd8fba05aU, - 0x690ce0eeU, 0x6dcdfd59U, 0x608edb80U, 0x644fc637U, - 0x7a089632U, 0x7ec98b85U, 0x738aad5cU, 0x774bb0ebU, - 0x4f040d56U, 0x4bc510e1U, 0x46863638U, 0x42472b8fU, - 0x5c007b8aU, 0x58c1663dU, 0x558240e4U, 0x51435d53U, - 0x251d3b9eU, 0x21dc2629U, 0x2c9f00f0U, 0x285e1d47U, - 0x36194d42U, 0x32d850f5U, 0x3f9b762cU, 0x3b5a6b9bU, - 0x0315d626U, 0x07d4cb91U, 0x0a97ed48U, 0x0e56f0ffU, - 0x1011a0faU, 0x14d0bd4dU, 0x19939b94U, 0x1d528623U, - 0xf12f560eU, 0xf5ee4bb9U, 0xf8ad6d60U, 0xfc6c70d7U, - 0xe22b20d2U, 0xe6ea3d65U, 0xeba91bbcU, 0xef68060bU, - 0xd727bbb6U, 0xd3e6a601U, 0xdea580d8U, 0xda649d6fU, - 0xc423cd6aU, 0xc0e2d0ddU, 0xcda1f604U, 0xc960ebb3U, - 0xbd3e8d7eU, 0xb9ff90c9U, 0xb4bcb610U, 0xb07daba7U, - 0xae3afba2U, 0xaafbe615U, 0xa7b8c0ccU, 0xa379dd7bU, - 0x9b3660c6U, 0x9ff77d71U, 0x92b45ba8U, 0x9675461fU, - 0x8832161aU, 0x8cf30badU, 0x81b02d74U, 0x857130c3U, - 0x5d8a9099U, 0x594b8d2eU, 0x5408abf7U, 0x50c9b640U, - 0x4e8ee645U, 0x4a4ffbf2U, 0x470cdd2bU, 0x43cdc09cU, - 0x7b827d21U, 0x7f436096U, 0x7200464fU, 0x76c15bf8U, - 0x68860bfdU, 0x6c47164aU, 0x61043093U, 0x65c52d24U, - 0x119b4be9U, 0x155a565eU, 0x18197087U, 0x1cd86d30U, - 0x029f3d35U, 0x065e2082U, 0x0b1d065bU, 0x0fdc1becU, - 0x3793a651U, 0x3352bbe6U, 0x3e119d3fU, 0x3ad08088U, - 0x2497d08dU, 0x2056cd3aU, 0x2d15ebe3U, 0x29d4f654U, - 0xc5a92679U, 0xc1683bceU, 0xcc2b1d17U, 0xc8ea00a0U, - 0xd6ad50a5U, 0xd26c4d12U, 0xdf2f6bcbU, 0xdbee767cU, - 0xe3a1cbc1U, 0xe760d676U, 0xea23f0afU, 0xeee2ed18U, - 0xf0a5bd1dU, 0xf464a0aaU, 0xf9278673U, 0xfde69bc4U, - 0x89b8fd09U, 0x8d79e0beU, 0x803ac667U, 0x84fbdbd0U, - 0x9abc8bd5U, 0x9e7d9662U, 0x933eb0bbU, 0x97ffad0cU, - 0xafb010b1U, 0xab710d06U, 0xa6322bdfU, 0xa2f33668U, - 0xbcb4666dU, 0xb8757bdaU, 0xb5365d03U, 0xb1f740b4U, - }; - - for (size_t i=0; i < data_len; i++) { - crc = table[(*data ^ (unsigned char)(crc >> 24)) & 0xFF] ^ (crc << 8); - data++; - } - return crc; -} - - -// Strangely, the PRS1 CRC32 appears to consider every byte a 32-bit wchar_t. -// Nothing like trying a bunch of encodings and CRC32 variants on PROP.TXT files -// until you find a winner. - -static crc32_t CRC32wchar(const unsigned char *data, size_t data_len, crc32_t crc=0xffffffffU) -{ - for (size_t i=0; i < data_len; i++) { - static unsigned char wch[4] = { 0, 0, 0, 0 }; - wch[3] = *data++; - crc = CRC32(wch, 4, crc); - } - return crc; -} - - QString ts(qint64 msecs) { // TODO: make this UTC so that tests don't vary by where they're run @@ -3014,338 +2868,6 @@ QList PRS1Loader::ParseFile(const QString & path) } -PRS1DataChunk::PRS1DataChunk(RawDataDevice & f, PRS1Loader* in_loader) : loader(in_loader) -{ - m_path = f.name(); -} - -PRS1DataChunk::~PRS1DataChunk() -{ - for (int i=0; i < m_parsedData.count(); i++) { - PRS1ParsedEvent* e = m_parsedData.at(i); - delete e; - } -} - - -PRS1DataChunk* PRS1DataChunk::ParseNext(RawDataDevice & f, PRS1Loader* loader) -{ - PRS1DataChunk* out_chunk = nullptr; - PRS1DataChunk* chunk = new PRS1DataChunk(f, loader); - - do { - // Parse the header and calculate its checksum. - bool ok = chunk->ReadHeader(f); - if (!ok) { - break; - } - - // Make sure the calculated checksum matches the stored checksum. - if (chunk->calcChecksum != chunk->storedChecksum) { - qWarning() << chunk->m_path << "header checksum calc" << chunk->calcChecksum << "!= stored" << chunk->storedChecksum; - break; - } - - // Read the block's data and calculate the block CRC. - ok = chunk->ReadData(f); - if (!ok) { - break; - } - - // Make sure the calculated CRC over the entire chunk (header and data) matches the stored CRC. - if (chunk->calcCrc != chunk->storedCrc) { - // Corrupt data block, warn about it. - qWarning() << chunk->m_path << "@" << chunk->m_filepos << "block CRC calc" << hex << chunk->calcCrc << "!= stored" << hex << chunk->storedCrc; - - // TODO: When this happens, it's usually because the chunk was truncated and another chunk header - // exists within the blockSize bytes. In theory it should be possible to rewing and resync by - // looking for another chunk header with the same fileVersion, htype, family, familyVersion, and - // ext (blockSize and other fields could vary). - // - // But this is quite rare, so for now we bail on the rest of the file. - break; - } - - // Only return the chunk if it has passed all tests above. - out_chunk = chunk; - } while (false); - - if (out_chunk == nullptr) delete chunk; - return out_chunk; -} - - -bool PRS1DataChunk::ReadHeader(RawDataDevice & f) -{ - bool ok = false; - do { - // Read common header fields. - this->m_filepos = f.pos(); - this->m_header = f.read(15); - if (this->m_header.size() != 15) { - if (this->m_header.size() == 0) { - qWarning() << this->m_path << "empty, skipping"; - } else { - qWarning() << this->m_path << "file too short?"; - } - break; - } - - unsigned char * header = (unsigned char *)this->m_header.data(); - this->fileVersion = header[0]; // Correlates to DataFileVersion in PROP[erties].TXT, only 2 or 3 has ever been observed - this->blockSize = (header[2] << 8) | header[1]; - this->htype = header[3]; // 00 = normal, 01=waveform - this->family = header[4]; - this->familyVersion = header[5]; - this->ext = header[6]; - this->sessionid = (header[10] << 24) | (header[9] << 16) | (header[8] << 8) | header[7]; - this->timestamp = (header[14] << 24) | (header[13] << 16) | (header[12] << 8) | header[11]; - - // Do a few early sanity checks before any variable-length header data. - if (this->blockSize == 0) { - qWarning() << this->m_path << "@" << hex << this->m_filepos << "blocksize 0, skipping remainder of file"; - break; - } - if (this->fileVersion < 2 || this->fileVersion > 3) { - if (this->m_filepos > 0) { - qWarning() << this->m_path << "@" << hex << this->m_filepos << "corrupt PRS1 header, skipping remainder of file"; - } else { - qWarning() << this->m_path << "unsupported PRS1 header version" << this->fileVersion; - } - break; - } - if (this->htype != PRS1_HTYPE_NORMAL && this->htype != PRS1_HTYPE_INTERVAL) { - qWarning() << this->m_path << "unexpected htype:" << this->htype; - break; - } - - // Read format-specific variable-length header data. - bool hdr_ok = false; - if (this->htype != PRS1_HTYPE_INTERVAL) { // Not just waveforms: the 1160P uses this for its .002 events file. - // Not a waveform/interval chunk - switch (this->fileVersion) { - case 2: - hdr_ok = ReadNormalHeaderV2(f); - break; - case 3: - hdr_ok = ReadNormalHeaderV3(f); - break; - default: - //hdr_ok remains false, warning is above - break; - } - } else { - // Waveform/interval chunk - hdr_ok = ReadWaveformHeader(f); - } - if (!hdr_ok) { - break; - } - - // The 8bit checksum comes at the end. - QByteArray checksum = f.read(1); - if (checksum.size() < 1) { - qWarning() << this->m_path << "read error header checksum"; - break; - } - this->storedChecksum = checksum.data()[0]; - - // Calculate 8bit additive header checksum. - header = (unsigned char *)this->m_header.data(); // important because its memory location could move - int header_size = this->m_header.size(); - quint8 achk=0; - for (int i=0; i < header_size; i++) { - achk += header[i]; - } - this->calcChecksum = achk; - - // Append the stored checksum to the raw data *after* calculating the checksum on the preceding data. - this->m_header.append(checksum); - - ok = true; - } while (false); - - return ok; -} - - -bool PRS1DataChunk::ReadNormalHeaderV2(RawDataDevice & /*f*/) -{ - this->m_headerblock = QByteArray(); - return true; // always OK -} - - -bool PRS1DataChunk::ReadNormalHeaderV3(RawDataDevice & f) -{ - bool ok = false; - unsigned char * header; - QByteArray headerB2; - - // This is a new machine, byte 15 is header data block length - // followed by variable, data byte pairs - do { - QByteArray extra = f.read(1); - if (extra.size() < 1) { - qWarning() << this->m_path << "read error extended header"; - break; - } - this->m_header.append(extra); - header = (unsigned char *)this->m_header.data(); - - int hdb_len = header[15]; - int hdb_size = hdb_len * 2; - - headerB2 = f.read(hdb_size); - if (headerB2.size() != hdb_size) { - qWarning() << this->m_path << "read error in extended header"; - break; - } - this->m_headerblock = headerB2; - - this->m_header.append(headerB2); - header = (unsigned char *)this->m_header.data(); - const unsigned char * hd = (unsigned char *)headerB2.constData(); - int pos = 0; - int recs = header[15]; - for (int i=0; ihblock[hd[pos]] = hd[pos+1]; - pos += 2; - } - - ok = true; - } while (false); - - return ok; -} - - -bool PRS1DataChunk::ReadWaveformHeader(RawDataDevice & f) -{ - bool ok = false; - unsigned char * header; - do { - // Read the fixed-length waveform header. - QByteArray extra = f.read(4); - if (extra.size() != 4) { - qWarning() << this->m_path << "read error in waveform header"; - break; - } - this->m_header.append(extra); - header = (unsigned char *)this->m_header.data(); - - // Parse the fixed-length portion. - this->interval_count = header[0x0f] | header[0x10] << 8; - this->interval_seconds = header[0x11]; // not always 1 after all - this->duration = this->interval_count * this->interval_seconds; // ??? the last entry doesn't always seem to be a full interval? - quint8 wvfm_signals = header[0x12]; - - // Read the variable-length data + trailing byte. - int ws_size = (this->fileVersion == 3) ? 4 : 3; - int sbsize = wvfm_signals * ws_size + 1; - - extra = f.read(sbsize); - if (extra.size() != sbsize) { - qWarning() << this->m_path << "read error in waveform header 2"; - break; - } - this->m_header.append(extra); - header = (unsigned char *)this->m_header.data(); - - // Parse the variable-length waveform information. - // TODO: move these checks into the parser, after the header checksum has been verified - // For now just skip them for the one known sample with a bad checksum. - if (this->sessionid == 268962649) return true; - - int pos = 0x13; - for (int i = 0; i < wvfm_signals; ++i) { - quint8 kind = header[pos]; - CHECK_VALUE(kind, i); // always seems to range from 0...wvfm_signals-1, alert if not - quint16 interleave = header[pos + 1] | header[pos + 2] << 8; // samples per interval - if (this->fileVersion == 2) { - this->waveformInfo.push_back(PRS1Waveform(interleave, kind)); - pos += 3; - } else if (this->fileVersion == 3) { - int always_8 = header[pos + 3]; // sample size in bits? - CHECK_VALUE(always_8, 8); - this->waveformInfo.push_back(PRS1Waveform(interleave, kind)); - pos += 4; - } - } - - // And the trailing byte, whatever it is. - int always_0 = header[pos]; - CHECK_VALUE(always_0, 0); - - ok = true; - } while (false); - - return ok; -} - - -bool PRS1DataChunk::ReadData(RawDataDevice & f) -{ - bool ok = false; - do { - // Read data block - int data_size = this->blockSize - this->m_header.size(); - if (data_size < 0) { - qWarning() << this->m_path << "chunk size smaller than header"; - break; - } - this->m_data = f.read(data_size); - if (this->m_data.size() < data_size) { - qWarning() << this->m_path << "less data in file than specified in header"; - break; - } - - // Extract the stored CRC from the data buffer and calculate the current CRC. - if (this->fileVersion==3) { - // The last 4 bytes contain a CRC32 checksum of the data. - if (!ExtractStoredCrc(4)) { - break; - } - this->calcCrc = CRC32wchar((unsigned char *)this->m_data.data(), this->m_data.size()); - } else { - // The last 2 bytes contain a CRC16 checksum of the data. - if (!ExtractStoredCrc(2)) { - break; - } - this->calcCrc = CRC16((unsigned char *)this->m_data.data(), this->m_data.size()); - } - - ok = true; - } while (false); - - return ok; -} - - -bool PRS1DataChunk::ExtractStoredCrc(int size) -{ - // Make sure there's enough data for the CRC. - int offset = this->m_data.size() - size; - if (offset < 0) { - qWarning() << this->m_path << "chunk truncated"; - return false; - } - - // Read the last 16- or 32-bit little-endian integer. - quint32 storedCrc = 0; - unsigned char* data = (unsigned char*)this->m_data.data(); - for (int i=0; i < size; i++) { - storedCrc |= data[offset+i] << (8*i); - } - this->storedCrc = storedCrc; - - // Drop the CRC from the data. - this->m_data.chop(size); - - return true; -} - - bool initialized = false; using namespace schema; diff --git a/oscar/SleepLib/loader_plugins/prs1_parser.cpp b/oscar/SleepLib/loader_plugins/prs1_parser.cpp index f8d2dd92..e14dce37 100644 --- a/oscar/SleepLib/loader_plugins/prs1_parser.cpp +++ b/oscar/SleepLib/loader_plugins/prs1_parser.cpp @@ -10,7 +10,7 @@ #include "prs1_parser.h" #include "prs1_loader.h" -#include +#include "rawdata.h" const PRS1ParsedEventType PRS1TidalVolumeEvent::TYPE; @@ -323,7 +323,7 @@ QMap PRS1SnoresAtPressureEvent::contents(void) //******************************************************************************************** // MARK: - -// MARK: Chunk parsing +// MARK: Parse chunk contents bool PRS1DataChunk::ParseCompliance(void) { @@ -917,3 +917,489 @@ void PRS1DataChunk::ParseTubingTypeV3(unsigned char type) } this->AddEvent(new PRS1ParsedSettingEvent(PRS1_SETTING_HOSE_DIAMETER, diam)); } + + +//******************************************************************************************** +// MARK: - +// MARK: Parse and verify chunk from stream + +typedef quint16 crc16_t; +typedef quint32 crc32_t; +static crc16_t CRC16(unsigned char * data, size_t data_len, crc16_t crc=0); +static crc32_t CRC32(const unsigned char *data, size_t data_len, crc32_t crc=0xffffffffU); +static crc32_t CRC32wchar(const unsigned char *data, size_t data_len, crc32_t crc=0xffffffffU); + +PRS1DataChunk::PRS1DataChunk(RawDataDevice & f, PRS1Loader* in_loader) : loader(in_loader) +{ + m_path = f.name(); +} + +PRS1DataChunk::~PRS1DataChunk() +{ + for (int i=0; i < m_parsedData.count(); i++) { + PRS1ParsedEvent* e = m_parsedData.at(i); + delete e; + } +} + + +PRS1DataChunk* PRS1DataChunk::ParseNext(RawDataDevice & f, PRS1Loader* loader) +{ + PRS1DataChunk* out_chunk = nullptr; + PRS1DataChunk* chunk = new PRS1DataChunk(f, loader); + + do { + // Parse the header and calculate its checksum. + bool ok = chunk->ReadHeader(f); + if (!ok) { + break; + } + + // Make sure the calculated checksum matches the stored checksum. + if (chunk->calcChecksum != chunk->storedChecksum) { + qWarning() << chunk->m_path << "header checksum calc" << chunk->calcChecksum << "!= stored" << chunk->storedChecksum; + break; + } + + // Read the block's data and calculate the block CRC. + ok = chunk->ReadData(f); + if (!ok) { + break; + } + + // Make sure the calculated CRC over the entire chunk (header and data) matches the stored CRC. + if (chunk->calcCrc != chunk->storedCrc) { + // Corrupt data block, warn about it. + qWarning() << chunk->m_path << "@" << chunk->m_filepos << "block CRC calc" << hex << chunk->calcCrc << "!= stored" << hex << chunk->storedCrc; + + // TODO: When this happens, it's usually because the chunk was truncated and another chunk header + // exists within the blockSize bytes. In theory it should be possible to rewing and resync by + // looking for another chunk header with the same fileVersion, htype, family, familyVersion, and + // ext (blockSize and other fields could vary). + // + // But this is quite rare, so for now we bail on the rest of the file. + break; + } + + // Only return the chunk if it has passed all tests above. + out_chunk = chunk; + } while (false); + + if (out_chunk == nullptr) delete chunk; + return out_chunk; +} + + +bool PRS1DataChunk::ReadHeader(RawDataDevice & f) +{ + bool ok = false; + do { + // Read common header fields. + this->m_filepos = f.pos(); + this->m_header = f.read(15); + if (this->m_header.size() != 15) { + if (this->m_header.size() == 0) { + qWarning() << this->m_path << "empty, skipping"; + } else { + qWarning() << this->m_path << "file too short?"; + } + break; + } + + unsigned char * header = (unsigned char *)this->m_header.data(); + this->fileVersion = header[0]; // Correlates to DataFileVersion in PROP[erties].TXT, only 2 or 3 has ever been observed + this->blockSize = (header[2] << 8) | header[1]; + this->htype = header[3]; // 00 = normal, 01=waveform + this->family = header[4]; + this->familyVersion = header[5]; + this->ext = header[6]; + this->sessionid = (header[10] << 24) | (header[9] << 16) | (header[8] << 8) | header[7]; + this->timestamp = (header[14] << 24) | (header[13] << 16) | (header[12] << 8) | header[11]; + + // Do a few early sanity checks before any variable-length header data. + if (this->blockSize == 0) { + qWarning() << this->m_path << "@" << hex << this->m_filepos << "blocksize 0, skipping remainder of file"; + break; + } + if (this->fileVersion < 2 || this->fileVersion > 3) { + if (this->m_filepos > 0) { + qWarning() << this->m_path << "@" << hex << this->m_filepos << "corrupt PRS1 header, skipping remainder of file"; + } else { + qWarning() << this->m_path << "unsupported PRS1 header version" << this->fileVersion; + } + break; + } + if (this->htype != PRS1_HTYPE_NORMAL && this->htype != PRS1_HTYPE_INTERVAL) { + qWarning() << this->m_path << "unexpected htype:" << this->htype; + break; + } + + // Read format-specific variable-length header data. + bool hdr_ok = false; + if (this->htype != PRS1_HTYPE_INTERVAL) { // Not just waveforms: the 1160P uses this for its .002 events file. + // Not a waveform/interval chunk + switch (this->fileVersion) { + case 2: + hdr_ok = ReadNormalHeaderV2(f); + break; + case 3: + hdr_ok = ReadNormalHeaderV3(f); + break; + default: + //hdr_ok remains false, warning is above + break; + } + } else { + // Waveform/interval chunk + hdr_ok = ReadWaveformHeader(f); + } + if (!hdr_ok) { + break; + } + + // The 8bit checksum comes at the end. + QByteArray checksum = f.read(1); + if (checksum.size() < 1) { + qWarning() << this->m_path << "read error header checksum"; + break; + } + this->storedChecksum = checksum.data()[0]; + + // Calculate 8bit additive header checksum. + header = (unsigned char *)this->m_header.data(); // important because its memory location could move + int header_size = this->m_header.size(); + quint8 achk=0; + for (int i=0; i < header_size; i++) { + achk += header[i]; + } + this->calcChecksum = achk; + + // Append the stored checksum to the raw data *after* calculating the checksum on the preceding data. + this->m_header.append(checksum); + + ok = true; + } while (false); + + return ok; +} + + +bool PRS1DataChunk::ReadNormalHeaderV2(RawDataDevice & /*f*/) +{ + this->m_headerblock = QByteArray(); + return true; // always OK +} + + +bool PRS1DataChunk::ReadNormalHeaderV3(RawDataDevice & f) +{ + bool ok = false; + unsigned char * header; + QByteArray headerB2; + + // This is a new machine, byte 15 is header data block length + // followed by variable, data byte pairs + do { + QByteArray extra = f.read(1); + if (extra.size() < 1) { + qWarning() << this->m_path << "read error extended header"; + break; + } + this->m_header.append(extra); + header = (unsigned char *)this->m_header.data(); + + int hdb_len = header[15]; + int hdb_size = hdb_len * 2; + + headerB2 = f.read(hdb_size); + if (headerB2.size() != hdb_size) { + qWarning() << this->m_path << "read error in extended header"; + break; + } + this->m_headerblock = headerB2; + + this->m_header.append(headerB2); + header = (unsigned char *)this->m_header.data(); + const unsigned char * hd = (unsigned char *)headerB2.constData(); + int pos = 0; + int recs = header[15]; + for (int i=0; ihblock[hd[pos]] = hd[pos+1]; + pos += 2; + } + + ok = true; + } while (false); + + return ok; +} + + +bool PRS1DataChunk::ReadWaveformHeader(RawDataDevice & f) +{ + bool ok = false; + unsigned char * header; + do { + // Read the fixed-length waveform header. + QByteArray extra = f.read(4); + if (extra.size() != 4) { + qWarning() << this->m_path << "read error in waveform header"; + break; + } + this->m_header.append(extra); + header = (unsigned char *)this->m_header.data(); + + // Parse the fixed-length portion. + this->interval_count = header[0x0f] | header[0x10] << 8; + this->interval_seconds = header[0x11]; // not always 1 after all + this->duration = this->interval_count * this->interval_seconds; // ??? the last entry doesn't always seem to be a full interval? + quint8 wvfm_signals = header[0x12]; + + // Read the variable-length data + trailing byte. + int ws_size = (this->fileVersion == 3) ? 4 : 3; + int sbsize = wvfm_signals * ws_size + 1; + + extra = f.read(sbsize); + if (extra.size() != sbsize) { + qWarning() << this->m_path << "read error in waveform header 2"; + break; + } + this->m_header.append(extra); + header = (unsigned char *)this->m_header.data(); + + // Parse the variable-length waveform information. + // TODO: move these checks into the parser, after the header checksum has been verified + // For now just skip them for the one known sample with a bad checksum. + if (this->sessionid == 268962649) return true; + + int pos = 0x13; + for (int i = 0; i < wvfm_signals; ++i) { + quint8 kind = header[pos]; + CHECK_VALUE(kind, i); // always seems to range from 0...wvfm_signals-1, alert if not + quint16 interleave = header[pos + 1] | header[pos + 2] << 8; // samples per interval + if (this->fileVersion == 2) { + this->waveformInfo.push_back(PRS1Waveform(interleave, kind)); + pos += 3; + } else if (this->fileVersion == 3) { + int always_8 = header[pos + 3]; // sample size in bits? + CHECK_VALUE(always_8, 8); + this->waveformInfo.push_back(PRS1Waveform(interleave, kind)); + pos += 4; + } + } + + // And the trailing byte, whatever it is. + int always_0 = header[pos]; + CHECK_VALUE(always_0, 0); + + ok = true; + } while (false); + + return ok; +} + + +bool PRS1DataChunk::ReadData(RawDataDevice & f) +{ + bool ok = false; + do { + // Read data block + int data_size = this->blockSize - this->m_header.size(); + if (data_size < 0) { + qWarning() << this->m_path << "chunk size smaller than header"; + break; + } + this->m_data = f.read(data_size); + if (this->m_data.size() < data_size) { + qWarning() << this->m_path << "less data in file than specified in header"; + break; + } + + // Extract the stored CRC from the data buffer and calculate the current CRC. + if (this->fileVersion==3) { + // The last 4 bytes contain a CRC32 checksum of the data. + if (!ExtractStoredCrc(4)) { + break; + } + this->calcCrc = CRC32wchar((unsigned char *)this->m_data.data(), this->m_data.size()); + } else { + // The last 2 bytes contain a CRC16 checksum of the data. + if (!ExtractStoredCrc(2)) { + break; + } + this->calcCrc = CRC16((unsigned char *)this->m_data.data(), this->m_data.size()); + } + + ok = true; + } while (false); + + return ok; +} + + +bool PRS1DataChunk::ExtractStoredCrc(int size) +{ + // Make sure there's enough data for the CRC. + int offset = this->m_data.size() - size; + if (offset < 0) { + qWarning() << this->m_path << "chunk truncated"; + return false; + } + + // Read the last 16- or 32-bit little-endian integer. + quint32 storedCrc = 0; + unsigned char* data = (unsigned char*)this->m_data.data(); + for (int i=0; i < size; i++) { + storedCrc |= data[offset+i] << (8*i); + } + this->storedCrc = storedCrc; + + // Drop the CRC from the data. + this->m_data.chop(size); + + return true; +} + + +// CRC-16/KERMIT, polynomial: 0x11021, bit reverse algorithm +// Table generated by crcmod (crc-kermit) + +typedef quint16 crc16_t; +static crc16_t CRC16(unsigned char * data, size_t data_len, crc16_t crc) +{ + static const crc16_t table[256] = { + 0x0000U, 0x1189U, 0x2312U, 0x329bU, 0x4624U, 0x57adU, 0x6536U, 0x74bfU, + 0x8c48U, 0x9dc1U, 0xaf5aU, 0xbed3U, 0xca6cU, 0xdbe5U, 0xe97eU, 0xf8f7U, + 0x1081U, 0x0108U, 0x3393U, 0x221aU, 0x56a5U, 0x472cU, 0x75b7U, 0x643eU, + 0x9cc9U, 0x8d40U, 0xbfdbU, 0xae52U, 0xdaedU, 0xcb64U, 0xf9ffU, 0xe876U, + 0x2102U, 0x308bU, 0x0210U, 0x1399U, 0x6726U, 0x76afU, 0x4434U, 0x55bdU, + 0xad4aU, 0xbcc3U, 0x8e58U, 0x9fd1U, 0xeb6eU, 0xfae7U, 0xc87cU, 0xd9f5U, + 0x3183U, 0x200aU, 0x1291U, 0x0318U, 0x77a7U, 0x662eU, 0x54b5U, 0x453cU, + 0xbdcbU, 0xac42U, 0x9ed9U, 0x8f50U, 0xfbefU, 0xea66U, 0xd8fdU, 0xc974U, + 0x4204U, 0x538dU, 0x6116U, 0x709fU, 0x0420U, 0x15a9U, 0x2732U, 0x36bbU, + 0xce4cU, 0xdfc5U, 0xed5eU, 0xfcd7U, 0x8868U, 0x99e1U, 0xab7aU, 0xbaf3U, + 0x5285U, 0x430cU, 0x7197U, 0x601eU, 0x14a1U, 0x0528U, 0x37b3U, 0x263aU, + 0xdecdU, 0xcf44U, 0xfddfU, 0xec56U, 0x98e9U, 0x8960U, 0xbbfbU, 0xaa72U, + 0x6306U, 0x728fU, 0x4014U, 0x519dU, 0x2522U, 0x34abU, 0x0630U, 0x17b9U, + 0xef4eU, 0xfec7U, 0xcc5cU, 0xddd5U, 0xa96aU, 0xb8e3U, 0x8a78U, 0x9bf1U, + 0x7387U, 0x620eU, 0x5095U, 0x411cU, 0x35a3U, 0x242aU, 0x16b1U, 0x0738U, + 0xffcfU, 0xee46U, 0xdcddU, 0xcd54U, 0xb9ebU, 0xa862U, 0x9af9U, 0x8b70U, + 0x8408U, 0x9581U, 0xa71aU, 0xb693U, 0xc22cU, 0xd3a5U, 0xe13eU, 0xf0b7U, + 0x0840U, 0x19c9U, 0x2b52U, 0x3adbU, 0x4e64U, 0x5fedU, 0x6d76U, 0x7cffU, + 0x9489U, 0x8500U, 0xb79bU, 0xa612U, 0xd2adU, 0xc324U, 0xf1bfU, 0xe036U, + 0x18c1U, 0x0948U, 0x3bd3U, 0x2a5aU, 0x5ee5U, 0x4f6cU, 0x7df7U, 0x6c7eU, + 0xa50aU, 0xb483U, 0x8618U, 0x9791U, 0xe32eU, 0xf2a7U, 0xc03cU, 0xd1b5U, + 0x2942U, 0x38cbU, 0x0a50U, 0x1bd9U, 0x6f66U, 0x7eefU, 0x4c74U, 0x5dfdU, + 0xb58bU, 0xa402U, 0x9699U, 0x8710U, 0xf3afU, 0xe226U, 0xd0bdU, 0xc134U, + 0x39c3U, 0x284aU, 0x1ad1U, 0x0b58U, 0x7fe7U, 0x6e6eU, 0x5cf5U, 0x4d7cU, + 0xc60cU, 0xd785U, 0xe51eU, 0xf497U, 0x8028U, 0x91a1U, 0xa33aU, 0xb2b3U, + 0x4a44U, 0x5bcdU, 0x6956U, 0x78dfU, 0x0c60U, 0x1de9U, 0x2f72U, 0x3efbU, + 0xd68dU, 0xc704U, 0xf59fU, 0xe416U, 0x90a9U, 0x8120U, 0xb3bbU, 0xa232U, + 0x5ac5U, 0x4b4cU, 0x79d7U, 0x685eU, 0x1ce1U, 0x0d68U, 0x3ff3U, 0x2e7aU, + 0xe70eU, 0xf687U, 0xc41cU, 0xd595U, 0xa12aU, 0xb0a3U, 0x8238U, 0x93b1U, + 0x6b46U, 0x7acfU, 0x4854U, 0x59ddU, 0x2d62U, 0x3cebU, 0x0e70U, 0x1ff9U, + 0xf78fU, 0xe606U, 0xd49dU, 0xc514U, 0xb1abU, 0xa022U, 0x92b9U, 0x8330U, + 0x7bc7U, 0x6a4eU, 0x58d5U, 0x495cU, 0x3de3U, 0x2c6aU, 0x1ef1U, 0x0f78U, + }; + + for (size_t i=0; i < data_len; i++) { + crc = table[(*data ^ (unsigned char)crc) & 0xFF] ^ (crc >> 8); + data++; + } + return crc; +} + + +// CRC-32/MPEG-2, polynomial: 0x104C11DB7 +// Table generated by crcmod (crc-32-mpeg) + +static crc32_t CRC32(const unsigned char *data, size_t data_len, crc32_t crc) +{ + static const crc32_t table[256] = { + 0x00000000U, 0x04c11db7U, 0x09823b6eU, 0x0d4326d9U, + 0x130476dcU, 0x17c56b6bU, 0x1a864db2U, 0x1e475005U, + 0x2608edb8U, 0x22c9f00fU, 0x2f8ad6d6U, 0x2b4bcb61U, + 0x350c9b64U, 0x31cd86d3U, 0x3c8ea00aU, 0x384fbdbdU, + 0x4c11db70U, 0x48d0c6c7U, 0x4593e01eU, 0x4152fda9U, + 0x5f15adacU, 0x5bd4b01bU, 0x569796c2U, 0x52568b75U, + 0x6a1936c8U, 0x6ed82b7fU, 0x639b0da6U, 0x675a1011U, + 0x791d4014U, 0x7ddc5da3U, 0x709f7b7aU, 0x745e66cdU, + 0x9823b6e0U, 0x9ce2ab57U, 0x91a18d8eU, 0x95609039U, + 0x8b27c03cU, 0x8fe6dd8bU, 0x82a5fb52U, 0x8664e6e5U, + 0xbe2b5b58U, 0xbaea46efU, 0xb7a96036U, 0xb3687d81U, + 0xad2f2d84U, 0xa9ee3033U, 0xa4ad16eaU, 0xa06c0b5dU, + 0xd4326d90U, 0xd0f37027U, 0xddb056feU, 0xd9714b49U, + 0xc7361b4cU, 0xc3f706fbU, 0xceb42022U, 0xca753d95U, + 0xf23a8028U, 0xf6fb9d9fU, 0xfbb8bb46U, 0xff79a6f1U, + 0xe13ef6f4U, 0xe5ffeb43U, 0xe8bccd9aU, 0xec7dd02dU, + 0x34867077U, 0x30476dc0U, 0x3d044b19U, 0x39c556aeU, + 0x278206abU, 0x23431b1cU, 0x2e003dc5U, 0x2ac12072U, + 0x128e9dcfU, 0x164f8078U, 0x1b0ca6a1U, 0x1fcdbb16U, + 0x018aeb13U, 0x054bf6a4U, 0x0808d07dU, 0x0cc9cdcaU, + 0x7897ab07U, 0x7c56b6b0U, 0x71159069U, 0x75d48ddeU, + 0x6b93dddbU, 0x6f52c06cU, 0x6211e6b5U, 0x66d0fb02U, + 0x5e9f46bfU, 0x5a5e5b08U, 0x571d7dd1U, 0x53dc6066U, + 0x4d9b3063U, 0x495a2dd4U, 0x44190b0dU, 0x40d816baU, + 0xaca5c697U, 0xa864db20U, 0xa527fdf9U, 0xa1e6e04eU, + 0xbfa1b04bU, 0xbb60adfcU, 0xb6238b25U, 0xb2e29692U, + 0x8aad2b2fU, 0x8e6c3698U, 0x832f1041U, 0x87ee0df6U, + 0x99a95df3U, 0x9d684044U, 0x902b669dU, 0x94ea7b2aU, + 0xe0b41de7U, 0xe4750050U, 0xe9362689U, 0xedf73b3eU, + 0xf3b06b3bU, 0xf771768cU, 0xfa325055U, 0xfef34de2U, + 0xc6bcf05fU, 0xc27dede8U, 0xcf3ecb31U, 0xcbffd686U, + 0xd5b88683U, 0xd1799b34U, 0xdc3abdedU, 0xd8fba05aU, + 0x690ce0eeU, 0x6dcdfd59U, 0x608edb80U, 0x644fc637U, + 0x7a089632U, 0x7ec98b85U, 0x738aad5cU, 0x774bb0ebU, + 0x4f040d56U, 0x4bc510e1U, 0x46863638U, 0x42472b8fU, + 0x5c007b8aU, 0x58c1663dU, 0x558240e4U, 0x51435d53U, + 0x251d3b9eU, 0x21dc2629U, 0x2c9f00f0U, 0x285e1d47U, + 0x36194d42U, 0x32d850f5U, 0x3f9b762cU, 0x3b5a6b9bU, + 0x0315d626U, 0x07d4cb91U, 0x0a97ed48U, 0x0e56f0ffU, + 0x1011a0faU, 0x14d0bd4dU, 0x19939b94U, 0x1d528623U, + 0xf12f560eU, 0xf5ee4bb9U, 0xf8ad6d60U, 0xfc6c70d7U, + 0xe22b20d2U, 0xe6ea3d65U, 0xeba91bbcU, 0xef68060bU, + 0xd727bbb6U, 0xd3e6a601U, 0xdea580d8U, 0xda649d6fU, + 0xc423cd6aU, 0xc0e2d0ddU, 0xcda1f604U, 0xc960ebb3U, + 0xbd3e8d7eU, 0xb9ff90c9U, 0xb4bcb610U, 0xb07daba7U, + 0xae3afba2U, 0xaafbe615U, 0xa7b8c0ccU, 0xa379dd7bU, + 0x9b3660c6U, 0x9ff77d71U, 0x92b45ba8U, 0x9675461fU, + 0x8832161aU, 0x8cf30badU, 0x81b02d74U, 0x857130c3U, + 0x5d8a9099U, 0x594b8d2eU, 0x5408abf7U, 0x50c9b640U, + 0x4e8ee645U, 0x4a4ffbf2U, 0x470cdd2bU, 0x43cdc09cU, + 0x7b827d21U, 0x7f436096U, 0x7200464fU, 0x76c15bf8U, + 0x68860bfdU, 0x6c47164aU, 0x61043093U, 0x65c52d24U, + 0x119b4be9U, 0x155a565eU, 0x18197087U, 0x1cd86d30U, + 0x029f3d35U, 0x065e2082U, 0x0b1d065bU, 0x0fdc1becU, + 0x3793a651U, 0x3352bbe6U, 0x3e119d3fU, 0x3ad08088U, + 0x2497d08dU, 0x2056cd3aU, 0x2d15ebe3U, 0x29d4f654U, + 0xc5a92679U, 0xc1683bceU, 0xcc2b1d17U, 0xc8ea00a0U, + 0xd6ad50a5U, 0xd26c4d12U, 0xdf2f6bcbU, 0xdbee767cU, + 0xe3a1cbc1U, 0xe760d676U, 0xea23f0afU, 0xeee2ed18U, + 0xf0a5bd1dU, 0xf464a0aaU, 0xf9278673U, 0xfde69bc4U, + 0x89b8fd09U, 0x8d79e0beU, 0x803ac667U, 0x84fbdbd0U, + 0x9abc8bd5U, 0x9e7d9662U, 0x933eb0bbU, 0x97ffad0cU, + 0xafb010b1U, 0xab710d06U, 0xa6322bdfU, 0xa2f33668U, + 0xbcb4666dU, 0xb8757bdaU, 0xb5365d03U, 0xb1f740b4U, + }; + + for (size_t i=0; i < data_len; i++) { + crc = table[(*data ^ (unsigned char)(crc >> 24)) & 0xFF] ^ (crc << 8); + data++; + } + return crc; +} + + +// Strangely, the PRS1 CRC32 appears to consider every byte a 32-bit wchar_t. +// Nothing like trying a bunch of encodings and CRC32 variants on PROP.TXT files +// until you find a winner. + +static crc32_t CRC32wchar(const unsigned char *data, size_t data_len, crc32_t crc) +{ + for (size_t i=0; i < data_len; i++) { + static unsigned char wch[4] = { 0, 0, 0, 0 }; + wch[3] = *data++; + crc = CRC32(wch, 4, crc); + } + return crc; +}