Add generic CSV reader class and switch ZEO loader to it.

No change in functionality for ZEO files.

Resolves crashing (assertion failure) on non-ZEO CSV files.

oscar/SleepLib/loader_plugins/zeo_loader.cpp

@@ -18,6 +18,7 @@
 #include <QTextStream>
 #include "zeo_loader.h"
 #include "SleepLib/machine.h"
+#include "csv.h"
 
 ZEOLoader::ZEOLoader()
 {
@@ -26,6 +27,7 @@ ZEOLoader::ZEOLoader()
 
 ZEOLoader::~ZEOLoader()
 {
+    closeCSV();
 }
 
 int ZEOLoader::Open(const QString & dirpath)
@@ -93,6 +95,7 @@ int ZEOLoader::OpenFile(const QString & filename)
         count++;
     }
     mach->Save();
+    closeCSV();
     return count;
 }
 
@@ -111,49 +114,46 @@ bool ZEOLoader::openCSV(const QString & filename)
         // not supported.
     }
 
-    text.setDevice(&file);
+    QStringList header;
-    QString headerdata = text.readLine();
+    csv = new CSVReader(file);
-    QStringList header = headerdata.split(",");
+    bool ok = csv->readRow(header);
+    if (!ok) {
+        qWarning() << "no header row";
+        return false;
+    }
+    csv->setFieldNames(header);
 
     MachineInfo info = newInfo();
     mach = p_profile->CreateMachine(info);
 
-    idxZQ = header.indexOf("ZQ");
+    return true;
+}
+
+void ZEOLoader::closeCSV()
+{
+    if (csv != nullptr) {
+        delete csv;
+        csv = nullptr;
+    }
+    if (file.isOpen()) {
+        file.close();
+    }
+}
+
 //    int idxTotalZ = header.indexOf("Total Z");
-    idxAwakenings = header.indexOf("Awakenings");
-    idxSG = header.indexOf("Sleep Graph");
-    idxDSG = header.indexOf("Detailed Sleep Graph");
-    idxTimeInWake = header.indexOf("Time in Wake");
-    idxTimeToZ = header.indexOf("Time to Z");
-    idxTimeInREM = header.indexOf("Time in REM");
-    idxTimeInLight = header.indexOf("Time in Light");
-    idxTimeInDeep = header.indexOf("Time in Deep");
-    idxStartOfNight = header.indexOf("Start of Night");
-    idxEndOfNight = header.indexOf("End of Night");
-    idxRiseTime = header.indexOf("Rise Time");
 //    int idxAlarmReason = header.indexOf("Alarm Reason");
 //    int idxSnoozeTime = header.indexOf("Snooze Time");
 //    int idxWakeTone = header.indexOf("Wake Tone");
 //    int idxWakeWindow = header.indexOf("Wake Window");
 //    int idxAlarmType = header.indexOf("Alarm Type");
-    idxFirstAlaramRing = header.indexOf("First Alarm Ring");
-    idxLastAlaramRing = header.indexOf("Last Alarm Ring");
-    idxFirstSnoozeTime = header.indexOf("First Snooze Time");
-    idxLastSnoozeTime = header.indexOf("Last Snooze Time");
-    idxSetAlarmTime = header.indexOf("Set Alarm Time");
-    idxMorningFeel = header.indexOf("Morning Feel");
-    idxFirmwareVersion = header.indexOf("Firmware Version");
-    idxMyZEOVersion = header.indexOf("My ZEO Version");
-
-    return true;
-}
-
 
 Session* ZEOLoader::readNextSession()
 {
+    if (csv == nullptr) {
+        qWarning() << "no CSV open!";
+        return nullptr;
+    }
     Session* sess = nullptr;
-    QString line;
-    QStringList linecomp;
     QDateTime start_of_night, end_of_night, rise_time;
     SessionID sid;
 
@@ -174,121 +174,97 @@ Session* ZEOLoader::readNextSession()
     bool ok;
     bool dodgy;
 
-    do {
+    QHash<QString,QString> row;
-        line = text.readLine();
+    while (csv->readRow(row)) {
         dodgy = false;
 
-        if (line.isEmpty()) { continue; }
+        ZQ = row["ZQ"].toInt(&ok);
-
-        linecomp = line.split(",");
-        ZQ = linecomp[idxZQ].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
 //        TotalZ = linecomp[idxTotalZ].toInt(&ok);
-
 //        if (!ok) { dodgy = true; }
 
-        TimeToZ = linecomp[idxTimeToZ].toInt(&ok);
+        TimeToZ = row["Time to Z"].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
-        TimeInWake = linecomp[idxTimeInWake].toInt(&ok);
+        TimeInWake = row["Time in Wake"].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
-        TimeInREM = linecomp[idxTimeInREM].toInt(&ok);
+        TimeInREM = row["Time in REM"].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
-        TimeInLight = linecomp[idxTimeInLight].toInt(&ok);
+        TimeInLight = row["Time in Light"].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
-        TimeInDeep = linecomp[idxTimeInDeep].toInt(&ok);
+        TimeInDeep = row["Time in Deep"].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
-        Awakenings = linecomp[idxAwakenings].toInt(&ok);
+        Awakenings = row["Awakenings"].toInt(&ok);
-
         if (!ok) { dodgy = true; }
 
-        start_of_night = readDateTime(linecomp[idxStartOfNight]);
+        start_of_night = readDateTime(row["Start of Night"]);
-
         if (!start_of_night.isValid()) { dodgy = true; }
 
-        end_of_night = readDateTime(linecomp[idxEndOfNight]);
+        end_of_night = readDateTime(row["End of Night"]);
-
         if (!end_of_night.isValid()) { dodgy = true; }
 
-        rise_time = readDateTime(linecomp[idxRiseTime]);
+        rise_time = readDateTime(row["Rise Time"]);
-
         if (!rise_time.isValid()) { dodgy = true; }
 
 //        AlarmReason = linecomp[idxAlarmReason].toInt(&ok);
-
 //        if (!ok) { dodgy = true; }
 
 //        SnoozeTime = linecomp[idxSnoozeTime].toInt(&ok);
-
 //        if (!ok) { dodgy = true; }
 
 //        WakeTone = linecomp[idxWakeTone].toInt(&ok);
-
 //        if (!ok) { dodgy = true; }
 
 //        WakeWindow = linecomp[idxWakeWindow].toInt(&ok);
-
 //        if (!ok) { dodgy = true; }
 
 //        AlarmType = linecomp[idxAlarmType].toInt(&ok);
-
 //        if (!ok) { dodgy = true; }
 
-        if (!linecomp[idxFirstAlaramRing].isEmpty()) {
+        if (!row["First Alarm Ring"].isEmpty()) {
-            FirstAlarmRing = readDateTime(linecomp[idxFirstAlaramRing]);
+            FirstAlarmRing = readDateTime(row["First Alarm Ring"]);
-
             if (!FirstAlarmRing.isValid()) { dodgy = true; }
         }
 
-        if (!linecomp[idxLastAlaramRing].isEmpty()) {
+        if (!row["Last Alarm Ring"].isEmpty()) {
-            LastAlarmRing = readDateTime(linecomp[idxLastAlaramRing]);
+            LastAlarmRing = readDateTime(row["Last Alarm Ring"]);
-
             if (!LastAlarmRing.isValid()) { dodgy = true; }
         }
 
-        if (!linecomp[idxFirstSnoozeTime].isEmpty()) {
+        if (!row["First Snooze Time"].isEmpty()) {
-            FirstSnoozeTime = readDateTime(linecomp[idxFirstSnoozeTime]);
+            FirstSnoozeTime = readDateTime(row["First Snooze Time"]);
 
             if (!FirstSnoozeTime.isValid()) { dodgy = true; }
         }
 
-        if (!linecomp[idxLastSnoozeTime].isEmpty()) {
+        if (!row["Last Snooze Time"].isEmpty()) {
-            LastSnoozeTime = readDateTime(linecomp[idxLastSnoozeTime]);
+            LastSnoozeTime = readDateTime(row["Last Snooze Time"]);
-
             if (!LastSnoozeTime.isValid()) { dodgy = true; }
         }
 
-        if (!linecomp[idxSetAlarmTime].isEmpty()) {
+        if (!row["Set Alarm Time"].isEmpty()) {
-            SetAlarmTime = readDateTime(linecomp[idxSetAlarmTime]);
+            SetAlarmTime = readDateTime(row["Set Alarm Time"]);
-
             if (!SetAlarmTime.isValid()) { dodgy = true; }
         }
 
-        MorningFeel = linecomp[idxMorningFeel].toInt(&ok);
+        MorningFeel = row["Morning Feel"].toInt(&ok);
-
         if (!ok) { MorningFeel = 0; }
 
-        FirmwareVersion = linecomp[idxFirmwareVersion];
+        FirmwareVersion = row["Firmware Version"];
 
-        if (idxMyZEOVersion >= 0) { MyZeoVersion = linecomp[idxMyZEOVersion]; }
+        MyZeoVersion = row["My ZEO Version"];
 
         if (dodgy) {
             continue;
         }
 
-        SG = linecomp[idxSG].split(" ");
+        SG = row["Sleep Graph"].split(" ");
-        DSG = linecomp[idxDSG].split(" ");
+        DSG = row["Detailed Sleep Graph"].split(" ");
 
         sid = start_of_night.toTime_t();
 
@@ -302,11 +278,10 @@ Session* ZEOLoader::readNextSession()
 
         sess = new Session(mach, sid);
         break;
-
+    };
-    } while (!line.isNull());
 
     if (sess) {
-        const int WindowSize = 30000;
+        const int WindowSize = 30 * 1000;
 
         sess->settings[ZEO_Awakenings] = Awakenings;
         sess->settings[ZEO_MorningFeel] = MorningFeel;
@@ -324,11 +299,9 @@ Session* ZEOLoader::readNextSession()
 
         for (int i = 0; i < DSG.size(); i++) {
             stage = DSG[i].toInt(&ok);
-
             if (ok) {
                 sleepstage->AddEvent(tt, stage);
             }
-
             tt += WindowSize;
         }
 

oscar/SleepLib/loader_plugins/zeo_loader.h

@@ -40,6 +40,7 @@ class ZEOLoader : public MachineLoader
     }
 
     bool openCSV(const QString & filename);
+    void closeCSV();
     Session* readNextSession();
 
   protected:
@@ -47,28 +48,8 @@ class ZEOLoader : public MachineLoader
 
   private:
     QFile file;
-    QTextStream text;
+    class CSVReader* csv;
     Machine *mach;
-    int idxZQ;
-    int idxAwakenings;
-    int idxSG;
-    int idxDSG;
-    int idxTimeInWake;
-    int idxTimeToZ;
-    int idxTimeInREM;
-    int idxTimeInLight;
-    int idxTimeInDeep;
-    int idxStartOfNight;
-    int idxEndOfNight;
-    int idxRiseTime;
-    int idxFirstAlaramRing;
-    int idxLastAlaramRing;
-    int idxFirstSnoozeTime;
-    int idxLastSnoozeTime;
-    int idxSetAlarmTime;
-    int idxMorningFeel;
-    int idxFirmwareVersion;
-    int idxMyZEOVersion;
 };
 
 #endif // ZEOLOADER_H

oscar/csv.cpp

@@ -0,0 +1,65 @@
+/* OSCAR CSV Reader Implementation
+ *
+ * Copyright (c) 2020 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#include "csv.h"
+#include <QDebug>
+
+CSVReader::CSVReader(QIODevice & input, const QString & delim, const QString & comment)
+    : m_stream(&input), m_delim(delim), m_comment(comment)
+{
+}
+
+void CSVReader::setFieldNames(QStringList & header)
+{
+    m_field_names = header;
+}
+
+// This is a very simplistic reader that splits lines on the delimiter and truncates
+// lines after the comment sequence (if specified). It doesn't do any quote handling.
+// If that's ultimately necessary, either rewrite it or subclass it.
+//
+// For a public domain version that handles RFC 4180, see:
+// https://stackoverflow.com/questions/27318631/parsing-through-a-csv-file-in-qt/40229435#40229435
+//
+bool CSVReader::readRow(QStringList & fields)
+{
+    QString line;
+    fields.clear();
+    
+    // Read until the next non-empty/non-comment line.
+    do {
+        line = m_stream.readLine();
+        if (line.isNull()) {
+            return false;
+        }
+        if (m_comment.isNull() == false) {
+            line = line.section(m_comment, 0, 0);
+        }
+    } while (line.isEmpty());
+
+    fields = line.split(m_delim);
+    return true;
+}
+
+bool CSVReader::readRow(QHash<QString,QString> & row)
+{
+    QStringList fields;
+    row.clear();
+    
+    if (!readRow(fields)) {
+        return false;
+    }
+    if (fields.size() > m_field_names.size()) {
+        qWarning() << "row has too many columns";
+        return false;
+    }
+    for (int i = 0; i < fields.size(); i++) {
+        row[m_field_names.at(i)] = fields.at(i);
+    }
+    return true;
+}

oscar/csv.h

@@ -0,0 +1,28 @@
+/* OSCAR CSV Reader Implementation
+ *
+ * Copyright (c) 2020 The OSCAR Team
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of the source code
+ * for more details. */
+
+#include <QTextStream>
+
+class CSVReader
+{
+public:
+    CSVReader(QIODevice & stream, const QString & delim=",", const QString & comment=QString());
+    virtual ~CSVReader() = default;
+    
+    QStringList readRow();
+    virtual bool readRow(QStringList & fields);  // override this for more complicated processing
+    void setFieldNames(QStringList & header);
+    bool readRow(QHash<QString,QString> & row);
+
+protected:
+    QTextStream m_stream;
+    QString m_delim;
+    QString m_comment;
+    QStringList m_field_names;
+};
+

oscar/oscar.pro

@@ -296,6 +296,7 @@ SOURCES += \
     SleepLib/loader_plugins/zeo_loader.cpp \
     zip.cpp \
     miniz.c \
+    csv.cpp \
     translation.cpp \
     statistics.cpp \
     oximeterimport.cpp \
@@ -371,6 +372,7 @@ HEADERS  += \
     SleepLib/loader_plugins/zeo_loader.h \
     zip.h \
     miniz.h \
+    csv.h \
     translation.h \
     statistics.h \
     oximeterimport.h \

oscar/tests/zeotests.cpp

@@ -48,6 +48,7 @@ static void parseAndEmitSessionYaml(const QString & path)
         if (count == 0) {
             qWarning() << "no sessions found";
         }
+        s_loader->closeCSV();
     } else {
         qWarning() << "unable to open file";
     }