chris/OSCAR-code
1
0
Fork 0
mirror of https://gitlab.com/pholy/OSCAR-code.git synced 2025-04-05 18:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite the Viatom header parsing to read all data and warn about unexpected data.

Browse Source 
It turns out the 2s-resolution files are actually 4s resolution with
each sample reported twice! Fixing that is next.
This commit is contained in:
sawinglogz 2020-01-24 20:10:41 -05:00
parent 21e7ae8b61
commit 61333aff07
2 changed files with 169 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
oscar/SleepLib/loader_plugins/viatom_loader.cpp
View File

@ -16,9 +16,13 @@
#include <QDir>
#include <QTextStream>
#include <QApplication>
#include <QMessageBox>
#include "viatom_loader.h"
#include "SleepLib/machine.h"
static QSet<QString> s_unexpectedMessages;
bool
ViatomLoader::Detect(const QString & path)
{
@ -38,14 +42,40 @@ ViatomLoader::Open(const QString & dirpath)
int
ViatomLoader::OpenFile(const QString & filename)
{
bool ok = false;
s_unexpectedMessages.clear();
qDebug() << "ViatomLoader::OpenFile(" << filename << ")";
Session* sess = ParseFile(filename);
if (sess != nullptr) {
if (sess == nullptr) {
QMessageBox::information(QApplication::activeWindow(),
QObject::tr("Unrecognized File"),
QObject::tr("This file does not appear to be a Viatom data file.") +"\n\n"+
QObject::tr("If it is, the developers will need a copy of this file so that future versions of OSCAR will be able to read it.")
,QMessageBox::Ok);
} else {
SaveSessionToDatabase(sess);
return true;
ok = true;
if (s_unexpectedMessages.count() > 0 && p_profile->session->warnOnUnexpectedData()) {
// Compare this to the list of messages previously seen for this machine
// and only alert if there are new ones.
QSet<QString> newMessages = s_unexpectedMessages - sess->machine()->previouslySeenUnexpectedData();
if (newMessages.count() > 0) {
// TODO: Rework the importer call structure so that this can become an
// emit statement to the appropriate import job.
QMessageBox::information(QApplication::activeWindow(),
QObject::tr("Untested Data"),
QObject::tr("Your Viatom device generated data that OSCAR has never seen before.") +"\n\n"+
QObject::tr("The imported data may not be entirely accurate, so the developers would like a copy of this file to make sure OSCAR is handling the data correctly.")
,QMessageBox::Ok);
sess->machine()->previouslySeenUnexpectedData() += newMessages;
}
}
}
return false;
return ok;
}
Session* ViatomLoader::ParseFile(const QString & filename)
@ -61,6 +91,8 @@ Session* ViatomLoader::ParseFile(const QString & filename)
return nullptr;
}
// TODO: Figure out what to do about machine ID. Right now OSCAR generates a random ID since we don't specify one.
// That means you won't be able to import multiple Viatom devices in a single session/profile.
MachineInfo info = newInfo();
Machine *mach = p_profile->CreateMachine(info);
Session *sess = mach->SessionExists(v.sessionid());
@ -75,14 +107,16 @@ Session* ViatomLoader::ParseFile(const QString & filename)
qDebug() << "Session" << v.sessionid() << "found...add data to it";
}
quint64 step = 2000; // records @ 2000ms (2 sec)
QList<ViatomFile::Record> records = v.ReadData();
quint64 step = v.duration() / records.size() * 1000L;
//CHECK_VALUES(step, 2000, 4000); // TODO: once ReadData deduplicates the records, there will only be 4000
EventList *ev_hr = sess->AddEventList(OXI_Pulse, EVL_Waveform, 1.0, 0.0, 0.0, 0.0, step);
EventList *ev_o2 = sess->AddEventList(OXI_SPO2, EVL_Waveform, 1.0, 0.0, 0.0, 0.0, step);
EventList *ev_mv = sess->AddEventList(POS_Motion, EVL_Waveform, 1.0, 0.0, 0.0, 0.0, step);
QList<ViatomFile::Record> records = v.ReadData();
// Import data
// TODO: Add support for multiple eventlists rather than holding the previous value.
ViatomFile::Record prev = { 99, 60, 0, 0, 0 };
for (auto & rec : records) {
if (rec.spo2 < 50 || rec.spo2 > 100) rec.spo2 = prev.spo2;
@ -98,9 +132,11 @@ Session* ViatomLoader::ParseFile(const QString & filename)
time_ms += step;
}
/*
qDebug() << "Read Viatom data from" << data_timestamp << "to" << (QDateTime::fromSecsSinceEpoch( time_ms / 1000L))
<< records.count() << "records"
<< ev_mv->Min() << "<=Motion<=" << ev_mv->Max();
*/
sess->setMin(OXI_Pulse, ev_hr->Min());
sess->setMax(OXI_Pulse, ev_hr->Max());
@ -139,43 +175,120 @@ ViatomLoader::Register()
// ===============================================================================================
static QString ts(qint64 msecs)
{
// TODO: make this UTC so that tests don't vary by where they're run
return QDateTime::fromMSecsSinceEpoch(msecs).toString(Qt::ISODate);
}
static QString dur(qint64 msecs)
{
qint64 s = msecs / 1000L;
int h = s / 3600; s -= h * 3600;
int m = s / 60; s -= m * 60;
return QString("%1:%2:%3")
.arg(h, 2, 10, QChar('0'))
.arg(m, 2, 10, QChar('0'))
.arg(s, 2, 10, QChar('0'));
}
// TODO: Merge this with PRS1 macros and generalize for all loaders.
#define UNEXPECTED_VALUE(SRC, VALS) { \
QString message = QString("%1:%2: %3 = %4 != %5").arg(__func__).arg(__LINE__).arg(#SRC).arg(SRC).arg(VALS); \
qWarning() << this->m_sessionid << message; \
s_unexpectedMessages += message; \
}
#define CHECK_VALUE(SRC, VAL) if ((SRC) != (VAL)) UNEXPECTED_VALUE(SRC, VAL)
#define CHECK_VALUES(SRC, VAL1, VAL2) if ((SRC) != (VAL1) && (SRC) != (VAL2)) UNEXPECTED_VALUE(SRC, #VAL1 " or " #VAL2)
// for more than 2 values, just write the test manually and use UNEXPECTED_VALUE if it fails
ViatomFile::ViatomFile(QFile & file) : m_file(file)
{
}
bool ViatomFile::ParseHeader()
{
QByteArray data;
qint64 filesize = m_file.size();
data = m_file.read(40);
QDataStream in(data);
in.setByteOrder(QDataStream::LittleEndian);
quint16 sig;
quint16 Y;
quint8 m,d,H,M,S;
in >> sig >> Y >> m >> d >> H >> M >> S;
if (sig != 0x0003 ||
(Y < 2015 || Y > 2040) ||
(m < 1 || m > 12) ||
(d < 1 || d > 31) ||
( H > 23) ||
( M > 60) ||
( S > 61)) {
qDebug() << m_file.fileName() << "does not appear to be a Viatom data file";
static const int HEADER_SIZE = 40;
QByteArray data = m_file.read(HEADER_SIZE);
if (data.size() < HEADER_SIZE) {
qDebug() << m_file.fileName() << "too short for a Viatom data file";
return false;
}
QDateTime data_timestamp = QDateTime(QDate(Y, m, d), QTime(H, M, S));
m_timestamp = data_timestamp.toMSecsSinceEpoch();
m_id = m_timestamp / 1000L;
const unsigned char* header = (const unsigned char*) data.constData();
int sig = header[0] | (header[1] << 8);
int year = header[2] | (header[3] << 8);
int month = header[4];
int day = header[5];
int hour = header[6];
int min = header[7];
int sec = header[8];
qDebug() << m_file.fileName() << "looks like a Viatom file, size" << filesize << "bytes signature" << sig
<< "start date/time" << data_timestamp << "(" << m_timestamp << ")";
if (sig != 0x0003) {
qDebug() << m_file.fileName() << "invalid signature for Viatom data file" << sig;
return false;
}
if ((year < 2015 || year > 2059) || (month < 1 || month > 12) || (day < 1 || day > 31) ||
(hour > 23) || (min > 59) || (sec > 59)) {
qDebug() << m_file.fileName() << "invalid timestamp in Viatom data file";
return false;
}
QDateTime data_timestamp = QDateTime(QDate(year, month, day), QTime(hour, min, sec));
m_timestamp = data_timestamp.toMSecsSinceEpoch();
m_sessionid = m_timestamp / 1000L;
int filesize = header[9] | (header[10] << 8); // possibly 32-bit
CHECK_VALUE(header[11], 0);
CHECK_VALUE(header[12], 0);
m_duration = header[13] | (header[14] << 8); // possibly 32-bit
CHECK_VALUE(header[15], 0);
CHECK_VALUE(header[16], 0);
//int spo2_avg = header[17];
//int spo2_min = header[18];
//int spo2_3pct = header[19]; // number of events
//int spo2_4pct = header[20]; // number of events
CHECK_VALUE(header[21], 0);
//int time_under_90pct = header[22]; // in seconds
CHECK_VALUE(header[23], 0);
//int events_under_90pct = header[24]; // number of distinct events
//float o2_score = header[25] * 0.1;
CHECK_VALUE(header[26], 0);
CHECK_VALUE(header[27], 0);
CHECK_VALUE(header[28], 0);
CHECK_VALUE(header[29], 0);
CHECK_VALUE(header[30], 0);
CHECK_VALUE(header[31], 0);
CHECK_VALUE(header[32], 0);
CHECK_VALUE(header[33], 0);
CHECK_VALUE(header[34], 0);
CHECK_VALUE(header[35], 0);
CHECK_VALUE(header[36], 0);
CHECK_VALUE(header[37], 0);
CHECK_VALUE(header[38], 0);
CHECK_VALUE(header[39], 0);
// Calculate timing resolution (in ms) of the data
qint64 datasize = m_file.size() - HEADER_SIZE;
m_record_count = datasize / RECORD_SIZE;
m_resolution = m_duration / m_record_count * 1000L;
if (m_resolution == 2000) {
// Interestingly the file size in the header corresponds the number of
// distinct samples. These files actually double-report each sample!
// So this resolution isn't really the real one. The importer should
// calculate resolution from duration / record count after reading the
// records, which will be deduplicated.
CHECK_VALUE(filesize, ((m_file.size() - HEADER_SIZE) / 2) + HEADER_SIZE);
} else {
CHECK_VALUE(filesize, m_file.size());
}
CHECK_VALUES(m_resolution, 2000, 4000);
CHECK_VALUE(datasize % RECORD_SIZE, 0);
CHECK_VALUE(m_duration % m_record_count, 0);
qDebug().noquote() << m_file.fileName() << ts(m_timestamp) << dur(m_duration * 1000L) << ":" << m_record_count << "records @" << m_resolution << "ms";
return true;
}
@ -195,5 +308,21 @@ QList<ViatomFile::Record> ViatomFile::ReadData()
records.append(rec);
} while (!in.atEnd());
// TODO: deduplicate the samples
/* It turns out 2s files are actually just double-reported samples!
if (m_resolution == 2000) {
CHECK_VALUE(records.size() % 2, 0);
for (int i = 0; i < records.size(); i += 2) {
auto & a = records.at(i);
auto & b = records.at(i+1);
CHECK_VALUE(a.spo2, b.spo2);
CHECK_VALUE(a.hr, b.hr);
CHECK_VALUE(a._unk1, b._unk1);
CHECK_VALUE(a.motion, b.motion);
CHECK_VALUE(a._unk2, b._unk2);
}
}
*/
return records;
}

9
oscar/SleepLib/loader_plugins/viatom_loader.h
View File

@ -62,13 +62,18 @@ public:
bool ParseHeader();
QList<Record> ReadData();
SessionID sessionid() const { return m_id; }
SessionID sessionid() const { return m_sessionid; }
quint64 timestamp() const { return m_timestamp; }
int duration() const { return m_duration; }
protected:
static const int RECORD_SIZE = 5;
QFile & m_file;
quint64 m_timestamp;
SessionID m_id;
int m_duration;
int m_record_count;
int m_resolution;
SessionID m_sessionid;
};
#endif // VIATOMLOADER_H