Consistent multi-file import for non-CPAP loaders (Viatom, Somnopose, Zeo, Dreem)

Htmldocs/release_notes.html

@@ -27,6 +27,8 @@
       <li>[new] Add Bulgarian translation; update other languages.</li>
       <li>[new] Improve Somnopose import options.</li>
       <li>[new] Purge Current Selected Day allows purge of each machine type separately</li>
+      <li>[new] Multi-file import for non-CPAP loaders (Somnopose, Viatom, Zeo, Dreem)</li>
+      <li>[new] Weight, BMI and Zombie history appear in statistics</li>
       <li>[fix] Correct calculation of average leak rate on Welcome page.</li>
       <li>[fix] Correct installation of non-English Release Notes on Windows.</li>
       <li>[fix] About/Credits page now offers Google translations to other languages.</li>

oscar/SleepLib/loader_plugins/dreem_loader.cpp

@@ -43,15 +43,6 @@ DreemLoader::Detect(const QString & path)
     return false;
 }
 
-int
-DreemLoader::Open(const QString & dirpath)
-{
-    qDebug() << "DreemLoader::Open(" << dirpath << ")";
-    // Dreem currently crams everything into a single file like Zeo did.
-    // See OpenFile.
-    return false;
-}
-
 int DreemLoader::OpenFile(const QString & filename)
 {
     if (!openCSV(filename)) {

oscar/SleepLib/loader_plugins/dreem_loader.h

@@ -25,8 +25,9 @@ class DreemLoader : public MachineLoader
 
     virtual bool Detect(const QString & path);
 
-    virtual int Open(const QString & path);
+    virtual int Open(const QString & path) { Q_UNUSED(path); return 0; } // Only for CPAP
     virtual int OpenFile(const QString & path);
+    virtual QStringList getNameFilter() { return QStringList("Dreem CSV File (*.csv)"); }
     static void Register();
 
     virtual int Version() { return dreem_data_version; }

oscar/SleepLib/loader_plugins/somnopose_loader.cpp

@@ -27,28 +27,6 @@ SomnoposeLoader::SomnoposeLoader()
 SomnoposeLoader::~SomnoposeLoader()
 {
 }
-int SomnoposeLoader::Open(const QString & dirpath)
-{
-    QString newpath;
-
-    QString dirtag = "somnopose";
-
-    QString path(dirpath);
-    path = path.replace("\\", "/");
-
-    if (path.toLower().endsWith("/" + dirtag)) {
-        return 0;
-        //newpath=path;
-    } else {
-        newpath = path + "/" + dirtag.toUpper();
-    }
-
-    //QString filename;
-
-    // Somnopose folder structure detection stuff here.
-
-    return 0; // number of machines affected
-}
 
 int SomnoposeLoader::OpenFile(const QString & filename)
 {
@@ -57,10 +35,10 @@ int SomnoposeLoader::OpenFile(const QString & filename)
     if (filename.toLower().endsWith(".csv")) {
         if (!file.open(QFile::ReadOnly)) {
             qDebug() << "Couldn't open Somnopose data file" << filename;
-            return 0;
+            return -1;
         }
     } else {
-        return 0;
+        return -1;
     }
 
     qDebug() << "Opening file" << filename;
@@ -107,12 +85,12 @@ int SomnoposeLoader::OpenFile(const QString & filename)
     // Check we have all fields available
     if (col_timestamp < 0) {
         qDebug() << "Header missing timestamp";
-        return 0;
+        return -1;
     }
 
     if ((col_inclination < 0) && (col_orientation < 0) && (col_movement < 0)) {
         qDebug() << "Header missing all of inclination, orientation, movement (at least one must be present)";
-        return 0;
+        return -1;
     }
 
     QDateTime epoch(QDate(2001, 1, 1));
@@ -169,7 +147,7 @@ int SomnoposeLoader::OpenFile(const QString & filename)
 
             if (mach->SessionExists(sid)) {
                 qDebug() << "File " << filename << " already loaded... skipping";
-                return -1; // Already imported
+                return 0; // Already imported
             }
 
             sess = new Session(mach, sid);
@@ -222,7 +200,7 @@ int SomnoposeLoader::OpenFile(const QString & filename)
         p_profile->StoreMachines();
     }
 
-    return true;
+    return 1;
 }
 
 

oscar/SleepLib/loader_plugins/somnopose_loader.h

@@ -26,8 +26,9 @@ class SomnoposeLoader : public MachineLoader
 
     virtual bool Detect(const QString & path) { Q_UNUSED(path); return false; }  // bypass autoscanner
 
-    virtual int Open(const QString & path);
+    virtual int Open(const QString & path) { Q_UNUSED(path); return 0; } // Only for CPAP
     virtual int OpenFile(const QString & filename);
+    virtual QStringList getNameFilter() { return QStringList("Somnopose CSV File (*.csv)"); }
     static void Register();
 
     virtual int Version() { return somnopose_data_version; }

oscar/SleepLib/loader_plugins/viatom_loader.cpp

@@ -34,33 +34,30 @@ ViatomLoader::Detect(const QString & path)
 }
 
 int
-ViatomLoader::Open(const QString & dirpath)
+ViatomLoader::Open(const QStringList & paths)
 {
-    qDebug() << "ViatomLoader::Open(" << dirpath << ")";
+    qDebug() << "ViatomLoader::Open(" << paths.join("; ") << ")";
     m_mach = nullptr;
     int imported = 0;
     int found = 0;
     s_unexpectedMessages.clear();
 
-    if (QFileInfo(dirpath).isDir()) {
+    int size = paths.size();
-        QDir dir(dirpath);
+    for (int i=0; i < size; i++) {
-        dir.setFilter(QDir::NoDotAndDotDot | QDir::Files | QDir::Hidden);
+        if (isAborted()) {
-        dir.setNameFilters(getNameFilter());
+            break;
-        dir.setSorting(QDir::Name);
-
-        for (auto & fi : dir.entryInfoList()) {
-            if (OpenFile(fi.canonicalFilePath())) {
-                imported++;
-            }
-            found++;
         }
-    }
-    else {
         // This filename has already been filtered by QFileDialog.
-        if (OpenFile(dirpath)) {
+        int ok = OpenFile(paths[i]);
+        if (ok > 0) {
             imported++;
+        } else if (ok < 0) {
+            // Stop on error...
+            break;
         }
         found++;
+        emit setProgressValue(i+1);
+        QCoreApplication::processEvents();
     }
 
     if (!found) {
@@ -91,25 +88,30 @@ ViatomLoader::Open(const QString & dirpath)
         }
     }
 
-    return imported;
+    return found;
 }
 
-bool ViatomLoader::OpenFile(const QString & filename)
+int ViatomLoader::OpenFile(const QString & filename)
 {
     Machine* mach = nullptr;
+    bool existing = false;
 
-    Session* sess = ParseFile(filename);
+    Session* sess = ParseFile(filename, &existing);
     if (sess) {
         SaveSessionToDatabase(sess);
         mach = sess->machine();
         m_mach = mach;
+        return 1;
     }
 
-    return mach != nullptr;
+    return existing ? 0 : -1; // -1 = error
 }
 
-Session* ViatomLoader::ParseFile(const QString & filename)
+Session* ViatomLoader::ParseFile(const QString & filename, bool *existing)
 {
+    if (existing) {
+        *existing = false;
+    }
     QFile file(filename);
     if (!file.open(QFile::ReadOnly)) {
         qDebug() << "Couldn't open Viatom data file" << filename;
@@ -136,6 +138,10 @@ Session* ViatomLoader::ParseFile(const QString & filename)
     if (mach->SessionExists(v.sessionid())) {
         // Skip already imported session
         //qDebug() << filename << "session already exists, skipping" << v.sessionid();
+        if (existing) {
+            // Inform the caller (if they are interested) that this session was already imported
+            *existing = true;
+        }
         return nullptr;
     }
 

oscar/SleepLib/loader_plugins/viatom_loader.h

@@ -28,8 +28,9 @@ class ViatomLoader : public MachineLoader
 
     virtual bool Detect(const QString & path);
 
-    virtual int Open(const QString & path);
+    virtual int Open(const QString & path) { Q_UNUSED(path); return 0; } // Only for CPAP
-    Session* ParseFile(const QString & filename);
+    virtual int Open(const QStringList & paths);
+    Session* ParseFile(const QString & filename, bool *existing=0);
 
     static void Register();
 
@@ -40,12 +41,12 @@ class ViatomLoader : public MachineLoader
         return MachineInfo(MT_OXIMETER, 0, viatom_class_name, QObject::tr("Viatom"), QString(), QString(), QString(), QObject::tr("Viatom Software"), QDateTime::currentDateTime(), viatom_data_version);
     }
 
-    QStringList getNameFilter();
+    virtual QStringList getNameFilter();
 
   //Machine *CreateMachine();
 
   protected:
-    bool OpenFile(const QString & filename);
+    int OpenFile(const QString & filename);
     void SaveSessionToDatabase(Session* session);
 
     void AddEvent(ChannelID channel, qint64 t, EventDataType value);

oscar/SleepLib/loader_plugins/zeo_loader.cpp

@@ -32,31 +32,6 @@ ZEOLoader::~ZEOLoader()
     closeCSV();
 }
 
-int ZEOLoader::Open(const QString & dirpath)
-{
-    QString newpath;
-
-    QString dirtag = "zeo";
-
-    // Could Scan the ZEO folder for a list of CSVs
-
-    QString path(dirpath);
-    path = path.replace("\\", "/");
-
-    if (path.toLower().endsWith("/" + dirtag)) {
-        return 0;
-        //newpath=path;
-    } else {
-        newpath = path + "/" + dirtag.toUpper();
-    }
-
-    //QString filename;
-
-    // ZEO folder structure detection stuff here.
-
-    return 0; // number of machines affected
-}
-
 /*15233: "Sleep Date"
 15234: "ZQ"
 15236: "Total Z"

oscar/SleepLib/loader_plugins/zeo_loader.h

@@ -27,8 +27,9 @@ class ZEOLoader : public MachineLoader
 
     virtual bool Detect(const QString &path) { Q_UNUSED(path); return false; }  // bypass autoscanner
 
-    virtual int Open(const QString & path);
+    virtual int Open(const QString & path) { Q_UNUSED(path); return 0; } // Only for CPAP
     virtual int OpenFile(const QString & filename);
+    virtual QStringList getNameFilter() { return QStringList("Zeo CSV File (*.csv)"); }
     static void Register();
 
     virtual int Version() { return zeo_data_version; }

oscar/SleepLib/machine_loader.cpp

@@ -316,3 +316,26 @@ bool compressFile(QString infile, QString outfile)
     return true;
 }
 
+int MachineLoader::Open(const QStringList & paths)
+{
+    int i, skipped = 0;
+    int size = paths.size();
+    for (i=0; i < size; i++) {
+        if (isAborted()) {
+            break;
+        }
+        QString filename = paths[i];
+
+        int res = OpenFile(filename);
+        if (res < 0) {
+            break;
+        }
+        if (res == 0) {
+            // Should we report on skipped count?
+            skipped++;
+        }
+        emit setProgressValue(i+1);
+        QCoreApplication::processEvents();
+    }
+    return i;
+}

oscar/SleepLib/machine_loader.h

@@ -56,9 +56,18 @@ class MachineLoader: public QObject
     //! \brief Override this to scan path and detect new machine data
     virtual int Open(const QString & path) = 0;
 
+    //! \brief Load all of the given files and update dialog with progress (for non-CPAP devices)
+    virtual int Open(const QStringList & paths);
+
+    //! \brief Load a specific (non-CPAP) file
+    virtual int OpenFile(const QString & path) { Q_UNUSED(path); return 0; }
+
     //! \brief Override to returns the Version number of this MachineLoader
     virtual int Version() = 0;
 
+    //! \brief Name filter for files for this loader
+    virtual QStringList getNameFilter() { return QStringList(""); }
+
     // !\\brief Used internally by loaders, override to return base MachineInfo record
     virtual MachineInfo newInfo() { return MachineInfo(); }
 

oscar/mainwindow.cpp

@@ -2311,64 +2311,14 @@ void MainWindow::doReprocessEvents()
 
 void MainWindow::on_actionImport_ZEO_Data_triggered()
 {
-    QFileDialog w;
-    w.setFileMode(QFileDialog::ExistingFiles);
-    w.setWindowFlags(this->windowFlags() & ~Qt::WindowContextHelpButtonHint);
-    w.setOption(QFileDialog::ShowDirsOnly, false);
-    w.setNameFilters(QStringList("Zeo CSV File (*.csv)"));
-
     ZEOLoader zeo;
-
+    importNonCPAP(zeo);
-    if (w.exec() == QFileDialog::Accepted) {
-        QString filename = w.selectedFiles()[0];
-
-        qDebug() << "Loading ZEO data from" << filename;
-        int c = zeo.OpenFile(filename);
-        if (c > 0) {
-            Notify(tr("Imported %1 ZEO session(s) from\n\n%2").arg(c).arg(filename), tr("Import Success"));
-            qDebug() << "Imported" << c << "ZEO sessions";
-            PopulatePurgeMenu();
-            if (overview) overview->ReloadGraphs();
-            if (welcome) welcome->refreshPage();
-        } else if (c == 0) {
-            Notify(tr("Already up to date with ZEO data at\n\n%1").arg(filename), tr("Up to date"));
-        } else {
-            Notify(tr("Couldn't find any valid ZEO CSV data at\n\n%1").arg(filename),tr("Import Problem"));
-        }
-
-        daily->LoadDate(daily->getDate());
-    }
 }
 
 void MainWindow::on_actionImport_Dreem_Data_triggered()
 {
-    QFileDialog w;
-    w.setFileMode(QFileDialog::ExistingFiles);
-    w.setWindowFlags(this->windowFlags() & ~Qt::WindowContextHelpButtonHint);
-    w.setOption(QFileDialog::ShowDirsOnly, false);
-    w.setNameFilters(QStringList("Dreem CSV File (*.csv)"));
-
     DreemLoader dreem;
-
+    importNonCPAP(dreem);
-    if (w.exec() == QFileDialog::Accepted) {
-        QString filename = w.selectedFiles()[0];
-
-        qDebug() << "Loading Dreem data from" << filename;
-        int c = dreem.OpenFile(filename);
-        if (c > 0) {
-            Notify(tr("Imported %1 Dreem session(s) from\n\n%2").arg(c).arg(filename), tr("Import Success"));
-            qDebug() << "Imported" << c << "Dreem sessions";
-            PopulatePurgeMenu();
-            if (overview) overview->ReloadGraphs();
-            if (welcome) welcome->refreshPage();
-        } else if (c == 0) {
-            Notify(tr("Already up to date with Dreem data at\n\n%1").arg(filename), tr("Up to date"));
-        } else {
-            Notify(tr("Couldn't find any valid Dreem CSV data at\n\n%1").arg(filename),tr("Import Problem"));
-        }
-
-        daily->LoadDate(daily->getDate());
-    }
 }
 
 void MainWindow::on_actionImport_RemStar_MSeries_Data_triggered()
@@ -2434,102 +2384,70 @@ void MainWindow::on_actionChange_Data_Folder_triggered()
     RestartApplication(false, "-d");
 }
 
-void MainWindow::on_actionImport_Somnopose_Data_triggered()
+void MainWindow::importNonCPAP(MachineLoader &loader)
 {
     QFileDialog w;
     w.setFileMode(QFileDialog::ExistingFiles);
     w.setWindowFlags(this->windowFlags() & ~Qt::WindowContextHelpButtonHint);
     w.setOption(QFileDialog::ShowDirsOnly, false);
+#if defined(Q_OS_WIN)
+    // Windows can't handle Viatom name filter - use non-native for all non-CPAP loaders.
     w.setOption(QFileDialog::DontUseNativeDialog, true);
-    w.setNameFilters(QStringList("Somnopause CSV File (*.csv)"));
+#endif
+    w.setNameFilters(loader.getNameFilter());
 
-    SomnoposeLoader somno;
     // Display progress if we have more than 1 file to load...
     ProgressDialog progress(this);
 
     if (w.exec() == QFileDialog::Accepted) {
-        int i, skipped = 0;
+        QStringList files = w.selectedFiles();
-        int size = w.selectedFiles().size();
+        int size = files.size();
         if (size > 1) {
             progress.setMessage(QObject::tr("Importing Sessions..."));
             progress.setProgressMax(size);
             progress.setProgressValue(0);
+            progress.addAbortButton();
             progress.setWindowModality(Qt::ApplicationModal);
+            connect(&loader, SIGNAL(setProgressValue(int)), &progress, SLOT(setProgressValue(int)));
+            connect(&progress, SIGNAL(abortClicked()), &loader, SLOT(abortImport()));
             progress.open();
             QCoreApplication::processEvents();
         }
-        for (i=0; i < size; i++) {
+        QString name = loader.loaderName();
-            QString filename = w.selectedFiles()[i];
+        int res = loader.Open(files);
-
+        if (size > 1) {
-            int res = somno.OpenFile(filename);
+            disconnect(&loader, SIGNAL(setProgressValue(int)), &progress, SLOT(setProgressValue(int)));
-            if (!res) {
+            disconnect(&progress, SIGNAL(abortClicked()), &loader, SLOT(abortImport()));
-                if (i == 0) {
+            progress.close();
-                    Notify(tr("There was a problem opening Somnopose Data File: ") + filename);
-                    return;
-                } else {
-                    Notify(tr("Somnopause Data Import of %1 file(s) complete").arg(i) + "\n\n" +
-                           tr("There was a problem opening Somnopose Data File: ") + filename,
-                           tr("Somnopose Import Partial Success"));
-                    break;
-                }
-            }
-            if (res < 0) {
-                // Should we report on skipped count?
-                skipped++;
-            }
-            progress.setProgressValue(i+1);
             QCoreApplication::processEvents();
         }
-
+        if (res == 0) {
-        if (i == size) {
+            Notify(tr("There was a problem opening %1 Data File: %2").arg(name, files[0]));
-            Notify(tr("Somnopause Data Import complete"));
+            return;
+        } else if (res < size){
+            Notify(tr("%1 Data Import of %2 file(s) complete").arg(name).arg(res) + "\n\n" +
+                   tr("There was a problem opening %1 Data File: %2").arg(name, files[res]),
+                   tr("%1 Import Partial Success").arg(name));
+        } else {
+            Notify(tr("%1 Data Import complete").arg(name));
         }
         PopulatePurgeMenu();
         if (overview) overview->ReloadGraphs();
         if (welcome) welcome->refreshPage();
         daily->LoadDate(daily->getDate());
     }
+}
 
+void MainWindow::on_actionImport_Somnopose_Data_triggered()
+{
+    SomnoposeLoader somno;
+    importNonCPAP(somno);
 }
 
 void MainWindow::on_actionImport_Viatom_Data_triggered()
 {
     ViatomLoader viatom;
-
+    importNonCPAP(viatom);
-    QFileDialog w;
-    w.setFileMode(QFileDialog::AnyFile);
-    w.setWindowFlags(this->windowFlags() & ~Qt::WindowContextHelpButtonHint);
-    w.setOption(QFileDialog::ShowDirsOnly, false);
-    w.setNameFilters(viatom.getNameFilter());
-#if defined(Q_OS_WIN)
-    // Windows can't handle this name filter.
-    w.setOption(QFileDialog::DontUseNativeDialog, true);
-    // And since the non-native dialog can't select both directories and files,
-    // it needs the following to enable selecting multiple files.
-    w.setFileMode(QFileDialog::ExistingFiles);
-#endif
-
-    if (w.exec() == QFileDialog::Accepted) {
-        QString filename = w.selectedFiles()[0];
-        if (w.selectedFiles().size() > 1) {
-            // The user selected multiple files in a directory, so use the parent directory as the filename.
-            filename = QFileInfo(filename).absoluteDir().canonicalPath();
-        }
-
-        int c = viatom.Open(filename);
-        if (c > 0) {
-            Notify(tr("Imported %1 oximetry session(s) from\n\n%2").arg(c).arg(filename), tr("Import Success"));
-            PopulatePurgeMenu();
-            if (overview) overview->ReloadGraphs();
-            if (welcome) welcome->refreshPage();
-        } else if (c == 0) {
-            Notify(tr("Already up to date with oximetry data at\n\n%1").arg(filename), tr("Up to date"));
-        } else {
-            Notify(tr("Couldn't find any valid data at\n\n%1").arg(filename),tr("Import Problem"));
-        }
-
-        daily->LoadDate(daily->getDate());
-    }
 }
 
 void MainWindow::GenerateStatistics()

oscar/mainwindow.h

@@ -378,6 +378,7 @@ private:
     void importCPAPDataCards(const QList<ImportPath> & datacards);
     void addMachineToMenu(Machine* mach, QMenu* menu);
     void purgeDay(MachineType type);
+    void importNonCPAP(MachineLoader &loader);
 
 //    QString getWelcomeHTML();
     void FreeSessions();