First pass at Dreem CSV loader.

Something's not quite right about the hypnogram timestamps, since
there are more than would fit within the start/stop times.

oscar/SleepLib/loader_plugins/dreem_loader.cpp

@@ -0,0 +1,258 @@
+/* SleepLib Dreem Loader Implementation
+ *
+ * Copyright (c) 2020 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+//********************************************************************************************
+// IMPORTANT!!!
+//********************************************************************************************
+// Please INCREMENT the dreem_data_version in dreem_loader.h when making changes to this loader
+// that change loader behaviour or modify channels.
+//********************************************************************************************
+
+#include <QDir>
+#include <QTextStream>
+#include <QApplication>
+#include <QMessageBox>
+#include "dreem_loader.h"
+#include "SleepLib/machine.h"
+#include "csv.h"
+
+static QSet<QString> s_unexpectedMessages;
+
+bool
+DreemLoader::Detect(const QString & path)
+{
+    // This is only used for CPAP machines, when detecting CPAP cards.
+    qDebug() << "DreemLoader::Detect(" << path << ")";
+    return false;
+}
+
+int
+DreemLoader::Open(const QString & dirpath)
+{
+    qDebug() << "DreemLoader::Open(" << dirpath << ")";
+    // Dreem currently crams everything into a single file like Zeo did.
+    // See OpenFile.
+    return false;
+}
+
+int DreemLoader::OpenFile(const QString & filename)
+{
+    if (!openCSV(filename)) {
+        closeCSV();
+        return -1;
+    }
+    int count = 0;
+    Session* sess;
+    while ((sess = readNextSession()) != nullptr) {
+        sess->SetChanged(true);
+        mach->AddSession(sess);
+        count++;
+    }
+    mach->Save();
+    closeCSV();
+    return count;
+}
+
+bool DreemLoader::openCSV(const QString & filename)
+{
+    file.setFileName(filename);
+
+    if (filename.toLower().endsWith(".csv")) {
+        if (!file.open(QFile::ReadOnly)) {
+            qDebug() << "Couldn't open Dreem file" << filename;
+            return false;
+        }
+    } else {
+        return false;
+    }
+
+    QStringList header;
+    csv = new CSVReader(file, ";", "#");
+    bool ok = csv->readRow(header);
+    if (!ok) {
+        qWarning() << "no header row";
+        return false;
+    }
+    csv->setFieldNames(header);
+
+    MachineInfo info = newInfo();
+    mach = p_profile->CreateMachine(info);
+
+    return true;
+}
+
+void DreemLoader::closeCSV()
+{
+    if (csv != nullptr) {
+        delete csv;
+        csv = nullptr;
+    }
+    if (file.isOpen()) {
+        file.close();
+    }
+}
+
+
+
+const QStringList s_sleepStageLabels = { "WAKE", "REM", "Light", "Deep" };
+
+Session* DreemLoader::readNextSession()
+{
+    if (csv == nullptr) {
+        qWarning() << "no CSV open!";
+        return nullptr;
+    }
+    static QHash<const QString,int> s_sleepStages;
+    for (int i = 0; i < s_sleepStageLabels.size(); i++) {
+        const QString & label = s_sleepStageLabels[i];
+        s_sleepStages[label] = i+1;  // match ZEO sleep stages for now
+        // TODO: generalize sleep stage integers between Dreem and Zeo
+    }
+
+    Session* sess = nullptr;
+
+    QDateTime start_time, stop_time;
+    int sleep_onset, sleep_duration;
+    int light_sleep_duration, deep_sleep_duration, rem_duration, awakened_duration;
+    int awakenings, position_changes, average_hr, average_rr;
+    float sleep_efficiency;
+    QStringList hypnogram;
+
+    QHash<QString,QString> row;
+    while (csv->readRow(row)) {
+        SessionID sid;
+        invalid_fields = false;
+
+        start_time = readDateTime(row["Start Time"]);
+        if (start_time.isValid()) {
+            sid = start_time.toTime_t();
+            if (mach->SessionExists(sid)) {
+                continue;
+            }
+        }
+
+        // "Type" always seems to be "night"
+        stop_time = readDateTime(row["Stop Time"]);
+        sleep_onset = readDuration(row["Sleep Onset Duration"]);
+        sleep_duration = readDuration(row["Sleep Duration"]);
+        light_sleep_duration = readDuration(row["Light Sleep Duration"]);
+        deep_sleep_duration = readDuration(row["Deep Sleep Duration"]);
+        rem_duration = readDuration(row["REM Duration"]);
+        awakened_duration = readDuration(row["Wake After Sleep Onset Duration"]);
+        awakenings = readInt(row["Number of awakenings"]);
+        position_changes = readInt(row["Position Changes"]);
+        average_hr = readInt(row["Mean Heart Rate"]);
+        average_rr = readInt(row["Mean Respiration CPM"]);
+        // "Number of Stimulations" is 0 for US models
+        sleep_efficiency = readInt(row["Sleep efficiency"]) / 100.0;
+        
+        if (invalid_fields) {
+            continue;
+        }
+
+        QString h = row["Hypnogram"];  // with "[" at the beginning and "]" at the end
+        hypnogram = h.mid(1, h.length()-2).split(",");
+        if (hypnogram.size() == 0) {
+            continue;
+        }
+
+        sess = new Session(mach, sid);
+        break;
+    };
+
+    if (sess) {
+        const quint64 step = 30 * 1000;
+
+        // TODO: rename Zeo channels to be generic
+        sess->settings[ZEO_Awakenings] = awakenings;
+        sess->settings[ZEO_TimeToZ] = sleep_onset / 60;  // TODO: convert durations to seconds and update Zeo loader accordingly, also below
+        sess->settings[ZEO_ZQ] = int(sleep_efficiency * 100.0);  // TODO: ZQ may be better expressed as a percent?
+        sess->settings[ZEO_TimeInWake] = awakened_duration / 60;
+        sess->settings[ZEO_TimeInREM] = rem_duration / 60;
+        sess->settings[ZEO_TimeInLight] = light_sleep_duration / 60;
+        sess->settings[ZEO_TimeInDeep] = deep_sleep_duration / 60;
+        sess->settings[OXI_Pulse] = average_hr;
+        sess->settings[CPAP_RespRate] = average_rr;
+        // Dreem also provides:
+        // total sleep duration
+        // # position changes
+
+        qint64 st = qint64(start_time.toTime_t()) * 1000L;
+        qint64 last = qint64(stop_time.toTime_t()) * 1000L;
+        sess->really_set_first(st);
+
+        // It appears that the first sample occurs at start time and
+        // the second sample occurs at the next 30-second boundary.
+        //
+        // TODO: About half the time there are still too many samples?
+        qint64 tt = st;
+        qint64 second_sample_tt = ((tt + step - 1L) / step) * step;
+
+        EventList *sleepstage = sess->AddEventList(ZEO_SleepStage, EVL_Event, 1, 0, 0, 4);
+
+        for (int i = 0; i < hypnogram.size(); i++) {
+            auto & label = hypnogram.at(i);
+            if (s_sleepStages.contains(label)) {
+                int stage = s_sleepStages[label];
+
+                // It appears that the last sample occurs at the stop time.
+                if (tt > last) {
+                    if (i != hypnogram.size() - 1) {
+                        qWarning() << sess->session() << "more hypnogram samples than time" << tt << last;
+                    }
+                    tt = last;
+                }
+
+                sleepstage->AddEvent(tt, stage);
+            }
+            if (i == 0) {
+                tt = second_sample_tt;
+            } else {
+                tt += step;
+            }
+        }
+
+        sess->really_set_last(last);
+    }
+
+    return sess;
+}
+
+QDateTime DreemLoader::readDateTime(const QString & text)
+{
+    QDateTime dt = QDateTime::fromString(text, Qt::ISODate);
+    if (!dt.isValid()) invalid_fields = true;
+    return dt;
+}
+
+int DreemLoader::readDuration(const QString & text)
+{
+    QTime t = QTime::fromString(text, "H:mm:ss");
+    if (!t.isValid()) invalid_fields = true;
+    return t.msecsSinceStartOfDay() / 1000L;
+}
+
+int DreemLoader::readInt(const QString & text)
+{
+    bool ok;
+    int value = text.toInt(&ok);
+    if (!ok) invalid_fields = true;
+    return value;
+}
+
+static bool dreem_initialized = false;
+
+void DreemLoader::Register()
+{
+    if (dreem_initialized) { return; }
+
+    qDebug("Registering DreemLoader");
+    RegisterLoader(new DreemLoader());
+    dreem_initialized = true;
+}
+

oscar/SleepLib/loader_plugins/dreem_loader.h

@@ -0,0 +1,55 @@
+/* SleepLib Dreem Loader Header
+ *
+ * Copyright (c) 2020 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#ifndef DREEMLOADER_H
+#define DREEMLOADER_H
+
+#include "SleepLib/machine_loader.h"
+
+const QString dreem_class_name = "Dreem";
+const int dreem_data_version = 1;
+
+
+/*! \class DreemLoader
+*/
+class DreemLoader : public MachineLoader
+{
+  public:
+    DreemLoader() { m_type = MT_SLEEPSTAGE; }
+    virtual ~DreemLoader() { }
+
+    virtual bool Detect(const QString & path);
+
+    virtual int Open(const QString & path);
+    virtual int OpenFile(const QString & path);
+    static void Register();
+
+    virtual int Version() { return dreem_data_version; }
+    virtual const QString &loaderName() { return dreem_class_name; }
+
+    virtual MachineInfo newInfo() {
+        return MachineInfo(MT_SLEEPSTAGE, 0, dreem_class_name, QObject::tr("Dreem"), QString(), QString(), QString(), QObject::tr("Dreem"), QDateTime::currentDateTime(), dreem_data_version);
+    }
+    
+    bool openCSV(const QString & filename);
+    void closeCSV();
+    Session* readNextSession();
+
+  protected:
+    QDateTime readDateTime(const QString & text);
+    int readDuration(const QString & text);
+    int readInt(const QString & text);
+
+  private:
+    QFile file;
+    class CSVReader* csv;
+    Machine *mach;
+    bool invalid_fields;
+};
+
+#endif // DREEMLOADER_H

oscar/mainwindow.cpp

@@ -39,6 +39,7 @@
 
 // Custom loaders that don't autoscan..
 #include <SleepLib/loader_plugins/zeo_loader.h>
+#include <SleepLib/loader_plugins/dreem_loader.h>
 #include <SleepLib/loader_plugins/somnopose_loader.h>
 #include <SleepLib/loader_plugins/viatom_loader.h>
 
@@ -2322,8 +2323,34 @@ void MainWindow::on_actionImport_ZEO_Data_triggered()
 
         daily->LoadDate(daily->getDate());
     }
+}
 
+void MainWindow::on_actionImport_Dreem_Data_triggered()
+{
+    QFileDialog w;
+    w.setFileMode(QFileDialog::ExistingFiles);
+    w.setWindowFlags(this->windowFlags() & ~Qt::WindowContextHelpButtonHint);
+    w.setOption(QFileDialog::ShowDirsOnly, false);
+    w.setNameFilters(QStringList("Dreem CSV File (*.csv)"));
 
+    DreemLoader dreem;
+
+    if (w.exec() == QFileDialog::Accepted) {
+        QString filename = w.selectedFiles()[0];
+
+        qDebug() << "Loading Dreem data from" << filename;
+        int c = dreem.OpenFile(filename);
+        if (c > 0) {
+            Notify(tr("Imported %1 Dreem session(s) from\n\n%2").arg(c).arg(filename), tr("Import Success"));
+            qDebug() << "Imported" << c << "Dreem sessions";
+        } else if (c == 0) {
+            Notify(tr("Already up to date with Dreem data at\n\n%1").arg(filename), tr("Up to date"));
+        } else {
+            Notify(tr("Couldn't find any valid Dreem CSV data at\n\n%1").arg(filename),tr("Import Problem"));
+        }
+
+        daily->LoadDate(daily->getDate());
+    }
 }
 
 void MainWindow::on_actionImport_RemStar_MSeries_Data_triggered()

oscar/mainwindow.h

@@ -281,6 +281,8 @@ class MainWindow : public QMainWindow
 
     void on_actionImport_ZEO_Data_triggered();
 
+    void on_actionImport_Dreem_Data_triggered();
+
     void on_actionImport_RemStar_MSeries_Data_triggered();
 
     void on_actionSleep_Disorder_Terms_Glossary_triggered();

oscar/mainwindow.ui

@@ -2911,6 +2911,7 @@ p, li { white-space: pre-wrap; }
     <addaction name="actionView_Oximetry"/>
     <addaction name="separator"/>
     <addaction name="actionImport_ZEO_Data"/>
+    <addaction name="actionImport_Dreem_Data"/>
     <addaction name="actionImport_Somnopose_Data"/>
     <addaction name="actionImport_Viatom_Data"/>
     <addaction name="actionImport_RemStar_MSeries_Data"/>
@@ -3119,6 +3120,11 @@ p, li { white-space: pre-wrap; }
     <string>Import &amp;ZEO Data</string>
    </property>
   </action>
+  <action name="actionImport_Dreem_Data">
+   <property name="text">
+    <string>Import &amp;Dreem Data</string>
+   </property>
+  </action>
   <action name="actionImport_RemStar_MSeries_Data">
    <property name="text">
     <string>Import RemStar &amp;MSeries Data</string>

oscar/oscar.pro

@@ -286,6 +286,7 @@ SOURCES += \
     SleepLib/schema.cpp \
     SleepLib/session.cpp \
     SleepLib/loader_plugins/cms50_loader.cpp \
+    SleepLib/loader_plugins/dreem_loader.cpp \
     SleepLib/loader_plugins/icon_loader.cpp \
     SleepLib/loader_plugins/intellipap_loader.cpp \
     SleepLib/loader_plugins/mseries_loader.cpp \
@@ -362,6 +363,7 @@ HEADERS  += \
     SleepLib/schema.h \
     SleepLib/session.h \
     SleepLib/loader_plugins/cms50_loader.h \
+    SleepLib/loader_plugins/dreem_loader.h \
     SleepLib/loader_plugins/icon_loader.h \
     SleepLib/loader_plugins/intellipap_loader.h \
     SleepLib/loader_plugins/mseries_loader.h \
@@ -520,6 +522,7 @@ test {
         tests/sessiontests.cpp \
         tests/versiontests.cpp \
         tests/viatomtests.cpp \
+        tests/dreemtests.cpp \
         tests/zeotests.cpp
 
     HEADERS += \
@@ -529,6 +532,7 @@ test {
         tests/sessiontests.h \
         tests/versiontests.h \
         tests/viatomtests.h \
+        tests/dreemtests.h \
         tests/zeotests.h
 }
 

oscar/tests/dreemtests.cpp

@@ -0,0 +1,93 @@
+/* Dreem Unit Tests
+ *
+ * Copyright (c) 2020 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#include "dreemtests.h"
+#include "sessiontests.h"
+
+#define TESTDATA_PATH "./testdata/"
+
+static DreemLoader* s_loader = nullptr;
+static QString dreemOutputPath(const QString & inpath, int sid, const QString & suffix);
+
+void DreemTests::initTestCase(void)
+{
+    p_profile = new Profile(TESTDATA_PATH "profile/", false);
+
+    schema::init();
+    DreemLoader::Register();
+    s_loader = dynamic_cast<DreemLoader*>(lookupLoader(dreem_class_name));
+}
+
+void DreemTests::cleanupTestCase(void)
+{
+    delete p_profile;
+    p_profile = nullptr;
+}
+
+
+// ====================================================================================================
+
+static void parseAndEmitSessionYaml(const QString & path)
+{
+    qDebug() << path;
+
+    if (s_loader->openCSV(path)) {
+        int count = 0;
+        Session* session;
+        while ((session = s_loader->readNextSession()) != nullptr) {
+            QString outpath = dreemOutputPath(path, session->session(), "-session.yml");
+            SessionToYaml(outpath, session, true);
+            delete session;
+            count++;
+        }
+        if (count == 0) {
+            qWarning() << "no sessions found";
+        }
+        s_loader->closeCSV();
+    } else {
+        qWarning() << "unable to open file";
+    }
+}
+
+void DreemTests::testSessionsToYaml()
+{
+    static const QString root_path = TESTDATA_PATH "dreem/input/";
+
+    QDir root(root_path);
+    root.setFilter(QDir::NoDotAndDotDot | QDir::Dirs);
+    root.setSorting(QDir::Name);
+    for (auto & dir_info : root.entryInfoList()) {
+        QDir dir(dir_info.canonicalFilePath());
+        dir.setFilter(QDir::Files | QDir::Hidden);
+        dir.setNameFilters(QStringList("*.csv"));
+        dir.setSorting(QDir::Name);
+        for (auto & fi : dir.entryInfoList()) {
+            parseAndEmitSessionYaml(fi.canonicalFilePath());
+        }
+    }
+}
+
+
+// ====================================================================================================
+
+QString dreemOutputPath(const QString & inpath, int sid, const QString & suffix)
+{
+    // Output to dreem/output/DIR/FILENAME(-session.yml, etc.)
+    QFileInfo path(inpath);
+    QString basename = path.baseName();
+    QString foldername = path.dir().dirName();
+
+    QDir outdir(TESTDATA_PATH "dreem/output/" + foldername);
+    outdir.mkpath(".");
+    
+    QString filename = QString("%1-%2%3")
+                        .arg(basename)
+                        .arg(sid, 8, 10, QChar('0'))
+                        .arg(suffix);
+    return outdir.path() + QDir::separator() + filename;
+}

oscar/tests/dreemtests.h

@@ -0,0 +1,28 @@
+/* Dreem Unit Tests
+ *
+ * Copyright (c) 2020 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#ifndef DREEMTESTS_H
+#define DREEMTESTS_H
+
+#include "AutoTest.h"
+#include "../SleepLib/loader_plugins/dreem_loader.h"
+
+class DreemTests : public QObject
+{
+    Q_OBJECT
+ 
+private slots:
+    void initTestCase();
+    void testSessionsToYaml();
+    void cleanupTestCase();
+};
+
+DECLARE_TEST(DreemTests)
+
+#endif // DREEMTESTS_H
+

oscar/tests/sessiontests.cpp

@@ -74,6 +74,8 @@ static QString settingChannel(ChannelID i)
         CHANNELNAME(CPAP_PSMax);
         CHANNELNAME(CPAP_RampTime);
         CHANNELNAME(CPAP_RampPressure);
+        CHANNELNAME(CPAP_RespRate);
+        CHANNELNAME(OXI_Pulse);
         CHANNELNAME(PRS1_FlexMode);
         CHANNELNAME(PRS1_FlexLevel);
         CHANNELNAME(PRS1_HumidStatus);