First PRS1 loader regression test: walk through a directory of SD cards and generate YAML for each session.

This has already exposed many limitations, and possibly some memory trampling. Before this will work as
a true regression test, we'll need to address both of those, so that this produces reliable, reproducible
output.

oscar/oscar.pro

@@ -440,14 +440,16 @@ test {
 
     QT += testlib
     QT -= gui
-    CONFIG += console
+    CONFIG += console debug
     CONFIG -= app_bundle
 
     SOURCES += \
-        tests/prs1tests.cpp
+        tests/prs1tests.cpp \
+        tests/sessiontests.cpp
 
     HEADERS += \
         tests/AutoTest.h \
-        tests/prs1tests.h
+        tests/prs1tests.h \
+        tests/sessiontests.h
 }
 

oscar/tests/prs1tests.cpp

@@ -1,7 +1,28 @@
+/* PRS1 Unit Tests
+ *
+ * Copyright (c) 2019 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
 #include "prs1tests.h"
+#include "sessiontests.h"
+
+#define TESTDATA_PATH "./testdata/"
+
+static PRS1Loader* s_loader = nullptr;
+static void iterateTestCards(const QString & root, void (*action)(const QString &));
+static QString prs1OutputPath(const QString & inpath, const QString & serial, int session, const QString & suffix);
 
 void PRS1Tests::initTestCase(void)
 {
+    QString profile_path = TESTDATA_PATH "profile/";
+    Profiles::Create("test", &profile_path);
+
+    schema::init();
+    PRS1Loader::Register();
+    s_loader = dynamic_cast<PRS1Loader*>(lookupLoader(prs1_class_name));
 }
 
 void PRS1Tests::cleanupTestCase(void)
@@ -9,9 +30,104 @@ void PRS1Tests::cleanupTestCase(void)
 }
 
 
-void PRS1Tests::test1()
+void parseAndEmitSessionYaml(const QString & path)
 {
-    // TODO: emit test message to stdout
+    qDebug() << path;
-    qDebug("First test!");
+
+    // This mirrors the functional bits of PRS1Loader::OpenMachine.
+    // Maybe there's a clever way to add parameters to OpenMachine that
+    // would make it more amenable to automated tests. But for now
+    // something is better than nothing.
+
+    QStringList paths;
+    QString propertyfile;
+    int sessionid_base;
+    sessionid_base = s_loader->FindSessionDirsAndProperties(path, paths, propertyfile);
+
+    Machine *m = s_loader->CreateMachineFromProperties(propertyfile);
+    Q_ASSERT(m != nullptr);
+
+    s_loader->ScanFiles(paths, sessionid_base, m);
+    
+    // Each session now has a PRS1Import object in m_tasklist
+    QList<ImportTask*>::iterator i;
+    for (i = s_loader->m_tasklist.begin(); i != s_loader->m_tasklist.end(); i++) {
+        // Run the parser
+        PRS1Import* import = dynamic_cast<PRS1Import*>(*i);
+        import->ParseSession();
+        
+        // Emit the parsed session data to compare against our regression benchmarks
+        Session* session = import->session;
+        QString outpath = prs1OutputPath(path, m->serial(), session->session(), "-session.yml");
+        SessionToYaml(outpath, session);
+        
+        delete session;
+        //delete import;  // TODO: this crashes: there's a bug in the loader somewhere
+   }
 }
 
+void PRS1Tests::testSessionsToYaml()
+{
+    iterateTestCards(TESTDATA_PATH "prs1/input/", parseAndEmitSessionYaml);
+}
+
+
+// ====================================================================================================
+
+QString prs1OutputPath(const QString & inpath, const QString & serial, int session, const QString & suffix)
+{
+    // Output to prs1/output/FOLDER/SERIAL-000000(-session.yml, etc.)
+    QDir path(inpath);
+    QStringList pathlist = QDir::toNativeSeparators(inpath).split(QDir::separator(), QString::SkipEmptyParts);
+    pathlist.pop_back();  // drop serial number directory
+    pathlist.pop_back();  // drop P-Series directory
+    QString foldername = pathlist.last();
+
+    QDir outdir(TESTDATA_PATH "prs1/output/" + foldername);
+    outdir.mkpath(".");
+    
+    QString filename = QString("%1-%2%3")
+                        .arg(serial)
+                        .arg(session, 6, 10, QChar('0'))
+                        .arg(suffix);
+    return outdir.path() + QDir::separator() + filename;
+}
+
+void iterateTestCards(const QString & root, void (*action)(const QString &))
+{
+    QDir dir(root);
+    dir.setFilter(QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files | QDir::Hidden | QDir::NoSymLinks);
+    dir.setSorting(QDir::Name);
+    QFileInfoList flist = dir.entryInfoList();
+
+    // Look through each folder in the given root
+    for (int i = 0; i < flist.size(); i++) {
+        QFileInfo fi = flist.at(i);
+        if (fi.isDir()) {
+            // If it contains a P-Series folder, it's a PRS1 SD card
+            QDir pseries(fi.canonicalFilePath() + QDir::separator() + "P-Series");
+            if (pseries.exists()) {
+                pseries.setFilter(QDir::NoDotAndDotDot | QDir::Dirs | QDir::Files | QDir::Hidden | QDir::NoSymLinks);
+                pseries.setSorting(QDir::Name);
+                QFileInfoList plist = pseries.entryInfoList();
+
+                // Look for machine directories (containing a PROP.TXT or properties.txt)
+                for (int j = 0; j < plist.size(); j++) {
+                    QFileInfo pfi = plist.at(j);
+                    if (pfi.isDir()) {
+                        QString machinePath = pfi.canonicalFilePath();
+                        QDir machineDir(machinePath);
+                        QFileInfoList mlist = machineDir.entryInfoList();
+                        for (int k = 0; k < mlist.size(); k++) {
+                            QFileInfo mfi = mlist.at(k);
+                            if (QDir::match("PROP*.TXT", mfi.fileName())) {
+                                // Found a properties file, this is a machine folder
+                                action(machinePath);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

oscar/tests/prs1tests.h

@@ -1,17 +1,26 @@
+/* PRS1 Unit Tests
+ *
+ * Copyright (c) The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
 #ifndef PRS1TESTS_H
 #define PRS1TESTS_H
 
 #include "AutoTest.h"
+#include "../SleepLib/loader_plugins/prs1_loader.h"
 
 class PRS1Tests : public QObject
 {
- Q_OBJECT
+    Q_OBJECT
-
+ 
 private slots:
- void initTestCase();
+    void initTestCase();
- void test1();
+    void testSessionsToYaml();
- // void test2();
+    // void test2();
- void cleanupTestCase();
+    void cleanupTestCase();
 };
 
 DECLARE_TEST(PRS1Tests)

oscar/tests/sessiontests.cpp

@@ -0,0 +1,229 @@
+/* Session Testing Support
+ *
+ * Copyright (c) 2019 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#include <QFile>
+#include "sessiontests.h"
+
+static QString ts(qint64 msecs)
+{
+    return QDateTime::fromMSecsSinceEpoch(msecs).toString(Qt::ISODate);
+}
+
+static QString hex(int i)
+{
+    return QString("0x") + QString::number(i, 16).toUpper();
+}
+
+#define ENUMSTRING(ENUM) case ENUM: s = #ENUM; break
+static QString eventListTypeName(EventListType t)
+{
+    QString s;
+    switch (t) {
+        ENUMSTRING(EVL_Waveform);
+        ENUMSTRING(EVL_Event);
+        default:
+            s = hex(t);
+            qDebug() << qPrintable(s);
+    }
+    return s;
+}
+
+// ChannelIDs are not enums. Instead, they are global variables of the ChannelID type.
+// This allows definition of different IDs within different loader plugins, while
+// using Qt templates (such as QHash) that require a consistent data type for their key.
+//
+// Ideally there would be a central ChannelID registry class that could be queried
+// for names, make sure there aren't duplicate values, etc. For now we just fill the
+// below by hand.
+#define CHANNELNAME(CH) if (i == CH) { s = #CH; break; }
+extern ChannelID PRS1_TimedBreath, PRS1_HeatedTubing;
+
+static QString settingChannel(ChannelID i)
+{
+    QString s;
+    do {
+        CHANNELNAME(CPAP_Mode);
+        CHANNELNAME(CPAP_Pressure);
+        CHANNELNAME(CPAP_PressureMin);
+        CHANNELNAME(CPAP_PressureMax);
+        CHANNELNAME(CPAP_EPAP);
+        CHANNELNAME(CPAP_IPAP);
+        CHANNELNAME(CPAP_PS);
+        CHANNELNAME(CPAP_EPAPLo);
+        CHANNELNAME(CPAP_EPAPHi);
+        CHANNELNAME(CPAP_IPAPLo);
+        CHANNELNAME(CPAP_IPAPHi);
+        CHANNELNAME(CPAP_PSMin);
+        CHANNELNAME(CPAP_PSMax);
+        CHANNELNAME(CPAP_RampTime);
+        CHANNELNAME(CPAP_RampPressure);
+        CHANNELNAME(PRS1_FlexMode);
+        CHANNELNAME(PRS1_FlexLevel);
+        CHANNELNAME(PRS1_HumidStatus);
+        CHANNELNAME(PRS1_HeatedTubing);        
+        CHANNELNAME(PRS1_HumidLevel);
+        CHANNELNAME(PRS1_SysLock);
+        CHANNELNAME(PRS1_SysOneResistSet);
+        CHANNELNAME(PRS1_SysOneResistStat);
+        CHANNELNAME(PRS1_TimedBreath);
+        CHANNELNAME(PRS1_HoseDiam);
+        CHANNELNAME(PRS1_AutoOn);
+        CHANNELNAME(PRS1_AutoOff);
+        CHANNELNAME(PRS1_MaskAlert);
+        CHANNELNAME(PRS1_ShowAHI);
+        s = hex(i);
+        qDebug() << qPrintable(s);
+    } while(false);
+    return s;
+}
+
+static QString eventChannel(ChannelID i)
+{
+    QString s;
+    do {
+        CHANNELNAME(CPAP_Obstructive);
+        CHANNELNAME(CPAP_Hypopnea);
+        CHANNELNAME(CPAP_PB);
+        CHANNELNAME(CPAP_LeakTotal);
+        CHANNELNAME(CPAP_Leak);
+        CHANNELNAME(CPAP_LargeLeak);
+        CHANNELNAME(CPAP_IPAP);
+        CHANNELNAME(CPAP_EPAP);
+        CHANNELNAME(CPAP_PS);
+        CHANNELNAME(CPAP_IPAPLo);
+        CHANNELNAME(CPAP_IPAPHi);
+        CHANNELNAME(CPAP_RespRate);
+        CHANNELNAME(CPAP_PTB);
+        CHANNELNAME(PRS1_TimedBreath);
+        CHANNELNAME(CPAP_MinuteVent);
+        CHANNELNAME(CPAP_TidalVolume);
+        CHANNELNAME(CPAP_ClearAirway);
+        CHANNELNAME(CPAP_FlowLimit);
+        CHANNELNAME(CPAP_Snore);
+        CHANNELNAME(CPAP_VSnore);
+        CHANNELNAME(CPAP_VSnore2);
+        CHANNELNAME(CPAP_NRI);
+        CHANNELNAME(CPAP_RERA);
+        CHANNELNAME(OXI_Pulse);
+        CHANNELNAME(OXI_SPO2);
+        CHANNELNAME(PRS1_BND);
+        CHANNELNAME(CPAP_MaskPressureHi);
+        CHANNELNAME(CPAP_FlowRate);
+        CHANNELNAME(CPAP_Test1);
+        CHANNELNAME(CPAP_Test2);
+        CHANNELNAME(CPAP_PressurePulse);
+        CHANNELNAME(CPAP_Pressure);
+        CHANNELNAME(PRS1_00);
+        CHANNELNAME(PRS1_01);
+        CHANNELNAME(PRS1_08);
+        CHANNELNAME(PRS1_0A);
+        CHANNELNAME(PRS1_0B);
+        CHANNELNAME(PRS1_0C);
+        CHANNELNAME(PRS1_0E);
+        CHANNELNAME(PRS1_15);
+        CHANNELNAME(CPAP_BrokenSummary);
+        s = hex(i);
+        qDebug() << qPrintable(s);
+    } while(false);
+    return s;
+}
+
+static QString intList(EventStoreType* data, int count)
+{
+    QStringList l;
+    for (int i = 0; i < count; i++) {
+        l.push_back(QString::number(data[i]));
+    }
+    QString s = "[ " + l.join(",") + " ]";
+    return s;
+}
+
+static QString intList(quint32* data, int count)
+{
+    QStringList l;
+    for (int i = 0; i < count; i++) {
+        l.push_back(QString::number(data[i] / 1000));
+    }
+    QString s = "[ " + l.join(",") + " ]";
+    return s;
+}
+
+void SessionToYaml(QString filepath, Session* session)
+{
+    QFile file(filepath);
+    if (!file.open(QFile::WriteOnly | QFile::Truncate)) {
+        qDebug() << filepath;
+        Q_ASSERT(false);
+    }
+    QTextStream out(&file);
+
+    // TODO: We sometimes see invalid session IDs. Either memory is getting trampled or the file
+    // header has the wrong ID (or isn't getting parsed right). Track this down once we can test parsing.
+    if (session->session() > 2000) qDebug() << "memory trampled? session ID" << session->session();
+    
+    out << "session:" << endl;
+    out << "  id: " << session->session() << endl;
+    out << "  start: " << ts(session->first()) << endl;
+    out << "  end: " << ts(session->last()) << endl;
+
+    out << "  settings:" << endl;
+
+    // We can't get deterministic ordering from QHash iterators, so we need to create a list
+    // of sorted ChannelIDs.
+    QList<ChannelID> keys = session->settings.keys();
+    std::sort(keys.begin(), keys.end());
+    for (QList<ChannelID>::iterator key = keys.begin(); key != keys.end(); key++) {
+        out << "    " << settingChannel(*key) << ": " << session->settings[*key].toString() << endl;
+    }
+
+    out << "  events:" << endl;
+
+    keys = session->eventlist.keys();
+    std::sort(keys.begin(), keys.end());
+    for (QList<ChannelID>::iterator key = keys.begin(); key != keys.end(); key++) {
+        out << "    " << eventChannel(*key) << ": " << endl;
+
+        // Note that this is a vector of lists
+        QVector<EventList *> &ev = session->eventlist[*key];
+        int ev_size = ev.size();
+        
+        // TODO: See what this actually signifies. Some waveform data seems to have to multiple event lists,
+        // which might reflect blocks within the original files, or something else.
+        if (ev_size > 2) qDebug() << session->session() << eventChannel(*key) << "ev_size =" << ev_size;
+
+        for (int j = 0; j < ev_size; j++) {
+            EventList &e = *ev[j];
+            out << "    - count: "  << (qint32)e.count() << endl;
+            if (e.count() == 0)
+                continue;
+            out << "      first: " << ts(e.first()) << endl;
+            out << "      last: " << ts(e.last()) << endl;
+            out << "      type: " << eventListTypeName(e.type()) << endl;
+            out << "      rate: " << e.rate() << endl;
+            out << "      gain: " << e.gain() << endl;
+            out << "      offset: " << e.offset() << endl;
+            if (!e.dimension().isEmpty()) {
+                out << "      dimension: " << e.dimension() << endl;
+            }
+            out << "      data:" << endl;
+            out << "        min: " << e.Min() << endl;
+            out << "        max: " << e.Max() << endl;
+            out << "        raw: " << intList((EventStoreType*) e.m_data.data(), e.count()) << endl;
+            if (e.type() != EVL_Waveform) {
+                out << "        delta: " << intList((quint32*) e.m_time.data(), e.count()) << endl;
+            }
+            if (e.hasSecondField()) {
+                out << "      data2:" << endl;
+                out << "        min: " << e.min2() << endl;
+                out << "        max: " << e.max2() << endl;
+                out << "        raw: " << intList((EventStoreType*) e.m_data2.data(), e.count()) << endl;
+            }
+        }
+    }
+    file.close();
+}

oscar/tests/sessiontests.h

@@ -0,0 +1,16 @@
+/* Session Testing Support
+ *
+ * Copyright (c) 2019 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#ifndef SESSIONTESTS_H
+#define SESSIONTESTS_H
+
+#include "../SleepLib/session.h"
+
+void SessionToYaml(QString filepath, Session* session);
+
+#endif // SESSIONTESTS_H