From 32ffcc4f947df0db35cacabff9674b8a2ab2890f Mon Sep 17 00:00:00 2001 From: sawinglogz <3787776-sawinglogz@users.noreply.gitlab.com> Date: Wed, 29 Jan 2020 19:59:05 -0500 Subject: [PATCH] First pass at Dreem CSV loader. Something's not quite right about the hypnogram timestamps, since there are more than would fit within the start/stop times. --- .../SleepLib/loader_plugins/dreem_loader.cpp | 258 ++++++++++++++++++ oscar/SleepLib/loader_plugins/dreem_loader.h | 55 ++++ oscar/mainwindow.cpp | 27 ++ oscar/mainwindow.h | 2 + oscar/mainwindow.ui | 6 + oscar/oscar.pro | 4 + oscar/tests/dreemtests.cpp | 93 +++++++ oscar/tests/dreemtests.h | 28 ++ oscar/tests/sessiontests.cpp | 2 + 9 files changed, 475 insertions(+) create mode 100644 oscar/SleepLib/loader_plugins/dreem_loader.cpp create mode 100644 oscar/SleepLib/loader_plugins/dreem_loader.h create mode 100644 oscar/tests/dreemtests.cpp create mode 100644 oscar/tests/dreemtests.h diff --git a/oscar/SleepLib/loader_plugins/dreem_loader.cpp b/oscar/SleepLib/loader_plugins/dreem_loader.cpp new file mode 100644 index 00000000..ecda42e7 --- /dev/null +++ b/oscar/SleepLib/loader_plugins/dreem_loader.cpp @@ -0,0 +1,258 @@ +/* SleepLib Dreem Loader Implementation + * + * Copyright (c) 2020 The OSCAR Team + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of the source code + * for more details. */ + +//******************************************************************************************** +// IMPORTANT!!! +//******************************************************************************************** +// Please INCREMENT the dreem_data_version in dreem_loader.h when making changes to this loader +// that change loader behaviour or modify channels. +//******************************************************************************************** + +#include +#include +#include +#include +#include "dreem_loader.h" +#include "SleepLib/machine.h" +#include "csv.h" + +static QSet s_unexpectedMessages; + +bool +DreemLoader::Detect(const QString & path) +{ + // This is only used for CPAP machines, when detecting CPAP cards. + qDebug() << "DreemLoader::Detect(" << path << ")"; + return false; +} + +int +DreemLoader::Open(const QString & dirpath) +{ + qDebug() << "DreemLoader::Open(" << dirpath << ")"; + // Dreem currently crams everything into a single file like Zeo did. + // See OpenFile. + return false; +} + +int DreemLoader::OpenFile(const QString & filename) +{ + if (!openCSV(filename)) { + closeCSV(); + return -1; + } + int count = 0; + Session* sess; + while ((sess = readNextSession()) != nullptr) { + sess->SetChanged(true); + mach->AddSession(sess); + count++; + } + mach->Save(); + closeCSV(); + return count; +} + +bool DreemLoader::openCSV(const QString & filename) +{ + file.setFileName(filename); + + if (filename.toLower().endsWith(".csv")) { + if (!file.open(QFile::ReadOnly)) { + qDebug() << "Couldn't open Dreem file" << filename; + return false; + } + } else { + return false; + } + + QStringList header; + csv = new CSVReader(file, ";", "#"); + bool ok = csv->readRow(header); + if (!ok) { + qWarning() << "no header row"; + return false; + } + csv->setFieldNames(header); + + MachineInfo info = newInfo(); + mach = p_profile->CreateMachine(info); + + return true; +} + +void DreemLoader::closeCSV() +{ + if (csv != nullptr) { + delete csv; + csv = nullptr; + } + if (file.isOpen()) { + file.close(); + } +} + + + +const QStringList s_sleepStageLabels = { "WAKE", "REM", "Light", "Deep" }; + +Session* DreemLoader::readNextSession() +{ + if (csv == nullptr) { + qWarning() << "no CSV open!"; + return nullptr; + } + static QHash s_sleepStages; + for (int i = 0; i < s_sleepStageLabels.size(); i++) { + const QString & label = s_sleepStageLabels[i]; + s_sleepStages[label] = i+1; // match ZEO sleep stages for now + // TODO: generalize sleep stage integers between Dreem and Zeo + } + + Session* sess = nullptr; + + QDateTime start_time, stop_time; + int sleep_onset, sleep_duration; + int light_sleep_duration, deep_sleep_duration, rem_duration, awakened_duration; + int awakenings, position_changes, average_hr, average_rr; + float sleep_efficiency; + QStringList hypnogram; + + QHash row; + while (csv->readRow(row)) { + SessionID sid; + invalid_fields = false; + + start_time = readDateTime(row["Start Time"]); + if (start_time.isValid()) { + sid = start_time.toTime_t(); + if (mach->SessionExists(sid)) { + continue; + } + } + + // "Type" always seems to be "night" + stop_time = readDateTime(row["Stop Time"]); + sleep_onset = readDuration(row["Sleep Onset Duration"]); + sleep_duration = readDuration(row["Sleep Duration"]); + light_sleep_duration = readDuration(row["Light Sleep Duration"]); + deep_sleep_duration = readDuration(row["Deep Sleep Duration"]); + rem_duration = readDuration(row["REM Duration"]); + awakened_duration = readDuration(row["Wake After Sleep Onset Duration"]); + awakenings = readInt(row["Number of awakenings"]); + position_changes = readInt(row["Position Changes"]); + average_hr = readInt(row["Mean Heart Rate"]); + average_rr = readInt(row["Mean Respiration CPM"]); + // "Number of Stimulations" is 0 for US models + sleep_efficiency = readInt(row["Sleep efficiency"]) / 100.0; + + if (invalid_fields) { + continue; + } + + QString h = row["Hypnogram"]; // with "[" at the beginning and "]" at the end + hypnogram = h.mid(1, h.length()-2).split(","); + if (hypnogram.size() == 0) { + continue; + } + + sess = new Session(mach, sid); + break; + }; + + if (sess) { + const quint64 step = 30 * 1000; + + // TODO: rename Zeo channels to be generic + sess->settings[ZEO_Awakenings] = awakenings; + sess->settings[ZEO_TimeToZ] = sleep_onset / 60; // TODO: convert durations to seconds and update Zeo loader accordingly, also below + sess->settings[ZEO_ZQ] = int(sleep_efficiency * 100.0); // TODO: ZQ may be better expressed as a percent? + sess->settings[ZEO_TimeInWake] = awakened_duration / 60; + sess->settings[ZEO_TimeInREM] = rem_duration / 60; + sess->settings[ZEO_TimeInLight] = light_sleep_duration / 60; + sess->settings[ZEO_TimeInDeep] = deep_sleep_duration / 60; + sess->settings[OXI_Pulse] = average_hr; + sess->settings[CPAP_RespRate] = average_rr; + // Dreem also provides: + // total sleep duration + // # position changes + + qint64 st = qint64(start_time.toTime_t()) * 1000L; + qint64 last = qint64(stop_time.toTime_t()) * 1000L; + sess->really_set_first(st); + + // It appears that the first sample occurs at start time and + // the second sample occurs at the next 30-second boundary. + // + // TODO: About half the time there are still too many samples? + qint64 tt = st; + qint64 second_sample_tt = ((tt + step - 1L) / step) * step; + + EventList *sleepstage = sess->AddEventList(ZEO_SleepStage, EVL_Event, 1, 0, 0, 4); + + for (int i = 0; i < hypnogram.size(); i++) { + auto & label = hypnogram.at(i); + if (s_sleepStages.contains(label)) { + int stage = s_sleepStages[label]; + + // It appears that the last sample occurs at the stop time. + if (tt > last) { + if (i != hypnogram.size() - 1) { + qWarning() << sess->session() << "more hypnogram samples than time" << tt << last; + } + tt = last; + } + + sleepstage->AddEvent(tt, stage); + } + if (i == 0) { + tt = second_sample_tt; + } else { + tt += step; + } + } + + sess->really_set_last(last); + } + + return sess; +} + +QDateTime DreemLoader::readDateTime(const QString & text) +{ + QDateTime dt = QDateTime::fromString(text, Qt::ISODate); + if (!dt.isValid()) invalid_fields = true; + return dt; +} + +int DreemLoader::readDuration(const QString & text) +{ + QTime t = QTime::fromString(text, "H:mm:ss"); + if (!t.isValid()) invalid_fields = true; + return t.msecsSinceStartOfDay() / 1000L; +} + +int DreemLoader::readInt(const QString & text) +{ + bool ok; + int value = text.toInt(&ok); + if (!ok) invalid_fields = true; + return value; +} + +static bool dreem_initialized = false; + +void DreemLoader::Register() +{ + if (dreem_initialized) { return; } + + qDebug("Registering DreemLoader"); + RegisterLoader(new DreemLoader()); + dreem_initialized = true; +} + diff --git a/oscar/SleepLib/loader_plugins/dreem_loader.h b/oscar/SleepLib/loader_plugins/dreem_loader.h new file mode 100644 index 00000000..b6569d76 --- /dev/null +++ b/oscar/SleepLib/loader_plugins/dreem_loader.h @@ -0,0 +1,55 @@ +/* SleepLib Dreem Loader Header + * + * Copyright (c) 2020 The OSCAR Team + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of the source code + * for more details. */ + +#ifndef DREEMLOADER_H +#define DREEMLOADER_H + +#include "SleepLib/machine_loader.h" + +const QString dreem_class_name = "Dreem"; +const int dreem_data_version = 1; + + +/*! \class DreemLoader +*/ +class DreemLoader : public MachineLoader +{ + public: + DreemLoader() { m_type = MT_SLEEPSTAGE; } + virtual ~DreemLoader() { } + + virtual bool Detect(const QString & path); + + virtual int Open(const QString & path); + virtual int OpenFile(const QString & path); + static void Register(); + + virtual int Version() { return dreem_data_version; } + virtual const QString &loaderName() { return dreem_class_name; } + + virtual MachineInfo newInfo() { + return MachineInfo(MT_SLEEPSTAGE, 0, dreem_class_name, QObject::tr("Dreem"), QString(), QString(), QString(), QObject::tr("Dreem"), QDateTime::currentDateTime(), dreem_data_version); + } + + bool openCSV(const QString & filename); + void closeCSV(); + Session* readNextSession(); + + protected: + QDateTime readDateTime(const QString & text); + int readDuration(const QString & text); + int readInt(const QString & text); + + private: + QFile file; + class CSVReader* csv; + Machine *mach; + bool invalid_fields; +}; + +#endif // DREEMLOADER_H diff --git a/oscar/mainwindow.cpp b/oscar/mainwindow.cpp index 2006fafa..1512a48a 100644 --- a/oscar/mainwindow.cpp +++ b/oscar/mainwindow.cpp @@ -39,6 +39,7 @@ // Custom loaders that don't autoscan.. #include +#include #include #include @@ -2322,8 +2323,34 @@ void MainWindow::on_actionImport_ZEO_Data_triggered() daily->LoadDate(daily->getDate()); } +} +void MainWindow::on_actionImport_Dreem_Data_triggered() +{ + QFileDialog w; + w.setFileMode(QFileDialog::ExistingFiles); + w.setWindowFlags(this->windowFlags() & ~Qt::WindowContextHelpButtonHint); + w.setOption(QFileDialog::ShowDirsOnly, false); + w.setNameFilters(QStringList("Dreem CSV File (*.csv)")); + DreemLoader dreem; + + if (w.exec() == QFileDialog::Accepted) { + QString filename = w.selectedFiles()[0]; + + qDebug() << "Loading Dreem data from" << filename; + int c = dreem.OpenFile(filename); + if (c > 0) { + Notify(tr("Imported %1 Dreem session(s) from\n\n%2").arg(c).arg(filename), tr("Import Success")); + qDebug() << "Imported" << c << "Dreem sessions"; + } else if (c == 0) { + Notify(tr("Already up to date with Dreem data at\n\n%1").arg(filename), tr("Up to date")); + } else { + Notify(tr("Couldn't find any valid Dreem CSV data at\n\n%1").arg(filename),tr("Import Problem")); + } + + daily->LoadDate(daily->getDate()); + } } void MainWindow::on_actionImport_RemStar_MSeries_Data_triggered() diff --git a/oscar/mainwindow.h b/oscar/mainwindow.h index 730f14f9..682eddbd 100644 --- a/oscar/mainwindow.h +++ b/oscar/mainwindow.h @@ -281,6 +281,8 @@ class MainWindow : public QMainWindow void on_actionImport_ZEO_Data_triggered(); + void on_actionImport_Dreem_Data_triggered(); + void on_actionImport_RemStar_MSeries_Data_triggered(); void on_actionSleep_Disorder_Terms_Glossary_triggered(); diff --git a/oscar/mainwindow.ui b/oscar/mainwindow.ui index 52e452e0..7e836930 100644 --- a/oscar/mainwindow.ui +++ b/oscar/mainwindow.ui @@ -2911,6 +2911,7 @@ p, li { white-space: pre-wrap; } + @@ -3119,6 +3120,11 @@ p, li { white-space: pre-wrap; } Import &ZEO Data + + + Import &Dreem Data + + Import RemStar &MSeries Data diff --git a/oscar/oscar.pro b/oscar/oscar.pro index 287a28b8..dd45704c 100644 --- a/oscar/oscar.pro +++ b/oscar/oscar.pro @@ -286,6 +286,7 @@ SOURCES += \ SleepLib/schema.cpp \ SleepLib/session.cpp \ SleepLib/loader_plugins/cms50_loader.cpp \ + SleepLib/loader_plugins/dreem_loader.cpp \ SleepLib/loader_plugins/icon_loader.cpp \ SleepLib/loader_plugins/intellipap_loader.cpp \ SleepLib/loader_plugins/mseries_loader.cpp \ @@ -362,6 +363,7 @@ HEADERS += \ SleepLib/schema.h \ SleepLib/session.h \ SleepLib/loader_plugins/cms50_loader.h \ + SleepLib/loader_plugins/dreem_loader.h \ SleepLib/loader_plugins/icon_loader.h \ SleepLib/loader_plugins/intellipap_loader.h \ SleepLib/loader_plugins/mseries_loader.h \ @@ -520,6 +522,7 @@ test { tests/sessiontests.cpp \ tests/versiontests.cpp \ tests/viatomtests.cpp \ + tests/dreemtests.cpp \ tests/zeotests.cpp HEADERS += \ @@ -529,6 +532,7 @@ test { tests/sessiontests.h \ tests/versiontests.h \ tests/viatomtests.h \ + tests/dreemtests.h \ tests/zeotests.h } diff --git a/oscar/tests/dreemtests.cpp b/oscar/tests/dreemtests.cpp new file mode 100644 index 00000000..298ef83d --- /dev/null +++ b/oscar/tests/dreemtests.cpp @@ -0,0 +1,93 @@ +/* Dreem Unit Tests + * + * Copyright (c) 2020 The OSCAR Team + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of the source code + * for more details. */ + +#include "dreemtests.h" +#include "sessiontests.h" + +#define TESTDATA_PATH "./testdata/" + +static DreemLoader* s_loader = nullptr; +static QString dreemOutputPath(const QString & inpath, int sid, const QString & suffix); + +void DreemTests::initTestCase(void) +{ + p_profile = new Profile(TESTDATA_PATH "profile/", false); + + schema::init(); + DreemLoader::Register(); + s_loader = dynamic_cast(lookupLoader(dreem_class_name)); +} + +void DreemTests::cleanupTestCase(void) +{ + delete p_profile; + p_profile = nullptr; +} + + +// ==================================================================================================== + +static void parseAndEmitSessionYaml(const QString & path) +{ + qDebug() << path; + + if (s_loader->openCSV(path)) { + int count = 0; + Session* session; + while ((session = s_loader->readNextSession()) != nullptr) { + QString outpath = dreemOutputPath(path, session->session(), "-session.yml"); + SessionToYaml(outpath, session, true); + delete session; + count++; + } + if (count == 0) { + qWarning() << "no sessions found"; + } + s_loader->closeCSV(); + } else { + qWarning() << "unable to open file"; + } +} + +void DreemTests::testSessionsToYaml() +{ + static const QString root_path = TESTDATA_PATH "dreem/input/"; + + QDir root(root_path); + root.setFilter(QDir::NoDotAndDotDot | QDir::Dirs); + root.setSorting(QDir::Name); + for (auto & dir_info : root.entryInfoList()) { + QDir dir(dir_info.canonicalFilePath()); + dir.setFilter(QDir::Files | QDir::Hidden); + dir.setNameFilters(QStringList("*.csv")); + dir.setSorting(QDir::Name); + for (auto & fi : dir.entryInfoList()) { + parseAndEmitSessionYaml(fi.canonicalFilePath()); + } + } +} + + +// ==================================================================================================== + +QString dreemOutputPath(const QString & inpath, int sid, const QString & suffix) +{ + // Output to dreem/output/DIR/FILENAME(-session.yml, etc.) + QFileInfo path(inpath); + QString basename = path.baseName(); + QString foldername = path.dir().dirName(); + + QDir outdir(TESTDATA_PATH "dreem/output/" + foldername); + outdir.mkpath("."); + + QString filename = QString("%1-%2%3") + .arg(basename) + .arg(sid, 8, 10, QChar('0')) + .arg(suffix); + return outdir.path() + QDir::separator() + filename; +} diff --git a/oscar/tests/dreemtests.h b/oscar/tests/dreemtests.h new file mode 100644 index 00000000..4a73704d --- /dev/null +++ b/oscar/tests/dreemtests.h @@ -0,0 +1,28 @@ +/* Dreem Unit Tests + * + * Copyright (c) 2020 The OSCAR Team + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of the source code + * for more details. */ + +#ifndef DREEMTESTS_H +#define DREEMTESTS_H + +#include "AutoTest.h" +#include "../SleepLib/loader_plugins/dreem_loader.h" + +class DreemTests : public QObject +{ + Q_OBJECT + +private slots: + void initTestCase(); + void testSessionsToYaml(); + void cleanupTestCase(); +}; + +DECLARE_TEST(DreemTests) + +#endif // DREEMTESTS_H + diff --git a/oscar/tests/sessiontests.cpp b/oscar/tests/sessiontests.cpp index 5bb43b7c..19310a0a 100644 --- a/oscar/tests/sessiontests.cpp +++ b/oscar/tests/sessiontests.cpp @@ -74,6 +74,8 @@ static QString settingChannel(ChannelID i) CHANNELNAME(CPAP_PSMax); CHANNELNAME(CPAP_RampTime); CHANNELNAME(CPAP_RampPressure); + CHANNELNAME(CPAP_RespRate); + CHANNELNAME(OXI_Pulse); CHANNELNAME(PRS1_FlexMode); CHANNELNAME(PRS1_FlexLevel); CHANNELNAME(PRS1_HumidStatus);