Sanity restored to Percentile calculations, restored summary only loading, and trashed the need for the cache.day file. Thank you Robin Hoffman for the awesome idea.

SleepLib/day.cpp

@@ -141,15 +141,12 @@ EventDataType Day::percentile(ChannelID code,EventDataType percentile)
     QVector<EventDataType> ar;
     for (s=sessions.begin();s!=sessions.end();s++) {
         Session & sess=*(*s);
-        QHash<ChannelID,QVector<EventList *> >::iterator ei=sess.eventlist.find(code);
+        QHash<ChannelID,QHash<EventStoreType, EventStoreType> > ::iterator ei=sess.m_valuesummary.find(code);
-
+        if (ei==sess.m_valuesummary.end()) continue;
-        if (ei==sess.eventlist.end())
+        EventDataType gain=sess.m_gain[code];
-            continue;
+        for (QHash<EventStoreType, EventStoreType>::iterator i=ei.value().begin();i!=ei.value().end();i++) {
-        for (int e=0;e<ei.value().size();e++) {
+            for (int j=0;j<i.value();j++) {
-            EventList *ev=ei.value()[e];
+                ar.push_back(float(i.key())*gain);
-            //if ()
-            for (unsigned j=0;j<ev->count();j++) {
-                ar.push_back(ev->data(j));
             }
         }
     }
@@ -176,73 +173,9 @@ EventDataType Day::percentile(ChannelID code,EventDataType percentile)
     return val;
 }
 
-EventDataType Day::p90(ChannelID code) // The "average" p90.. this needs fixing.
+EventDataType Day::p90(ChannelID code)
 {
-    int size=sessions.size();
+    return percentile(code,0.90);
-
-    if (size==0) return 0;
-    else if (size==1) return sessions[0]->p90(code);
-
-    QHash<ChannelID,EventDataType>::iterator i=m_p90.find(code);
-    if (i!=m_p90.end())
-        return i.value();
-
-
-    QVector<Session *>::iterator s;
-
-    // Don't assume sessions are in order.
-
-    unsigned cnt=0,c;
-    EventDataType p;
-
-    QMap<EventDataType, unsigned> pmap;
-    QMap<EventDataType, float> tmap;
-    QMap<EventDataType, unsigned>::iterator pi;
-    for (s=sessions.begin();s!=sessions.end();s++) {
-        Session & sess=*(*s);
-        c=sess.count(code);
-        if (c>0) {
-            cnt+=c;
-            p=sess.p90(code); //percentile(code,0.9);
-            if (!pmap.contains(p)) {
-                pmap[p]=c;
-                tmap[p]=sess.hours();
-            } else {
-                pmap[p]+=c;
-                tmap[p]+=sess.hours();
-            }
-        }
-    }
-    EventDataType val;
-    size=pmap.size();
-    if (!size) {
-        m_p90[code]=val=0;
-        return val;
-    } else if (size==1) {
-        m_p90[code]=val=pmap.begin().key();
-        return val;
-    }
-
-    OpenEvents();
-    EventDataType realp90=percentile(code,0.9);
-
-    val=realp90;
-    /*double s0=0,s1=0,s2=0,s3=0;
-
-    for (pi=pmap.begin();pi!=pmap.end();pi++) {
-        s2=tmap[pi.key()];
-        s3=pi.value();
-        s0+=pi.key() * s3 * s2;
-        s1+=s3*s2;
-    }
-    if (s1==0)
-        return 0;
-
-    val=s0/s1;
-    qDebug() << first() << code << realp90 << val; */
-
-    m_p90[code]=val;
-    return val;
 }
 
 EventDataType Day::avg(ChannelID code)

SleepLib/day.h

@@ -71,13 +71,6 @@ public:
     //! \brief Returns if the cache contains SummaryType information about the requested code
     bool hasData(ChannelID code, SummaryType type);
 
-    //! \brief Cache for holding slow calculated 90th Percentile for this day
-    QHash<ChannelID, EventDataType> m_p90;
-
-    //EventDataType percentile(ChannelID mc,double percent);
-
-    // Note, the following convert to doubles without considering the consequences fully.?
-
     //! \brief Returns the Average of all Sessions setting 'code' for this day
     EventDataType settings_avg(ChannelID code);
 

SleepLib/machine.cpp

@@ -266,7 +266,7 @@ bool Machine::Load()
 
         if (sess->LoadSummary(s.value()[0])) {
              sess->SetEventFile(s.value()[1]);
-             sess->OpenEvents();
+             //sess->OpenEvents();
              AddSession(sess,profile);
         } else {
             qWarning() << "Error unpacking summary data";

SleepLib/profiles.cpp

@@ -63,46 +63,6 @@ Profile::Profile(QString path)
 }
 bool Profile::Save(QString filename)
 {
-    if (p_profile==this) {
-        QString cachefile=Get("{DataFolder}")+QDir::separator()+"cache.day";
-        QFile f(cachefile);
-        if (ExistsAndTrue("TrashDayCache"))  {
-            if (f.exists()) {
-                f.remove();
-            }
-        } else {
-            QMap<QDate,QList<Day *> >::iterator di;
-            QHash<MachineID,QMap<QDate,QHash<ChannelID, EventDataType> > > cache;
-
-            QHash<MachineID,QMap<QDate,QHash<ChannelID, EventDataType> > >::iterator ci;
-            for (di=daylist.begin();di!=daylist.end();di++) {
-                QDate date=di.key();
-                for (QList<Day *>::iterator d=di.value().begin();d!=di.value().end();d++) {
-                    Day *day=*d;
-                    MachineID mach=day->machine->id();
-                    QHash<ChannelID, EventDataType>::iterator i;
-
-                    for (i=day->m_p90.begin();i!=day->m_p90.end();i++) {
-                        cache[mach][date][i.key()]=day->m_p90[i.key()];
-                    }
-                }
-            }
-            if (f.open(QFile::WriteOnly)) {
-                QDataStream out(&f);
-                out.setVersion(QDataStream::Qt_4_6);
-                out.setByteOrder(QDataStream::LittleEndian);
-                quint16 size=cache.size();
-                out << size;
-                for (ci=cache.begin();ci!=cache.end();ci++) {
-                    quint32 mid=ci.key();
-                    out << mid;
-                    out << ci.value();
-                }
-                f.close();
-            }
-        }
-        (*this)["TrashDayCache"]=false;
-    }
     return Preferences::Save(filename);
 }
 
@@ -145,29 +105,6 @@ void Profile::LoadMachineData()
 {
     QHash<MachineID,QMap<QDate,QHash<ChannelID, EventDataType> > > cache;
 
-    QString filename=Get("{DataFolder}")+QDir::separator()+"cache.day";
-    QFile f(filename);
-    if (f.exists(filename) && (f.open(QFile::ReadOnly))) {
-        QDataStream in(&f);
-        in.setVersion(QDataStream::Qt_4_6);
-        in.setByteOrder(QDataStream::LittleEndian);
-
-        quint16 size;
-        quint32 mid;
-        in >> size;
-        for (int i=0;i<size;i++) {
-            in >> mid;
-            in >> cache[mid];
-        }
-        PROFILE.general->setRebuildCache(false);
-    } else {
-        if (mainwin) {
-            mainwin->Notify(QObject::tr("Caching session data, this may take a little while."));
-            PROFILE.general->setRebuildCache(true);
-
-            QApplication::processEvents();
-        }
-    }
     for (QHash<MachineID,Machine *>::iterator i=machlist.begin(); i!=machlist.end(); i++) {
         Machine *m=i.value();
 
@@ -197,18 +134,6 @@ void Profile::LoadMachineData()
             m->Load();
         }
     }
-    for (QMap<QDate,QList<Day *> >::iterator di=daylist.begin();di!=daylist.end();di++) {
-        for (QList<Day *>::iterator d=di.value().begin();d!=di.value().end();d++) {
-            Day *day=*d;
-            MachineID mid=day->machine->id();
-            if (cache.contains(mid)) {
-                if (cache[mid].contains(di.key())) {
-                    day->m_p90=cache[mid][di.key()];
-                }
-            }
-        }
-    }
-    // Load Day Cache here..
 }
 
 /**
@@ -798,6 +723,14 @@ EventDataType Profile::calcSettingsMax(ChannelID code, MachineType mt, QDate sta
     if (max<=-99999999) max=0;
     return max;
 }
+
+struct CountSummary {
+    CountSummary(EventStoreType v) :val(v), count(0), time(0) {}
+    EventStoreType val;
+    EventStoreType count;
+    quint32 time;
+};
+
 EventDataType Profile::calcPercentile(ChannelID code, EventDataType percent, MachineType mt, QDate start, QDate end)
 {
     if (!start.isValid()) start=LastDay(mt);
@@ -807,32 +740,58 @@ EventDataType Profile::calcPercentile(ChannelID code, EventDataType percent, Mac
 
     // This is one messy function.. It requires all data to be loaded.. :(
 
+    QHash<EventStoreType,EventStoreType> summary;
+    QHash<EventStoreType,EventStoreType>::iterator sumi;
     QVector<EventDataType> array;
+
+    QHash<ChannelID,QHash<EventStoreType, EventStoreType> >::iterator vsi;
+    float gain;
+    bool setgain=false;
     //double val=0,tmp,hours=0;
     do {
         Day * day=GetDay(date,mt);
         if (day) {
-            bool open=day->eventsLoaded();
-            if (!open)
-                day->OpenEvents();
             for (int i=0;i<day->size();i++) {
                 for (QVector<Session *>::iterator s=day->begin();s!=day->end();s++) {
-                    QHash<ChannelID,QVector<EventList *> >::iterator el=(*s)->eventlist.find(code);
+                    Session *sess=*s;
+                    gain=sess->m_gain[code];
+                    if (!gain) gain=1;
+                    setgain=true;
+                    vsi=sess->m_valuesummary.find(code);
+                    if (vsi==sess->m_valuesummary.end()) continue;
+
+                    QHash<EventStoreType, EventStoreType> & vsum=vsi.value();
+
+                    for (QHash<EventStoreType, EventStoreType>::iterator k=vsum.begin();k!=vsum.end();k++) {
+                        for (int z=0;z<k.value();z++) {
+                            array.push_back(float(k.key())*gain);
+                        }
+                        /*sumi=summary.find(k.key());
+                        if (sumi==summary.end()) summary[k.key()]=k.value();
+                        else {
+                            sumi.value()+=k.value();
+                        }*/
+                    }
+                    /*QHash<ChannelID,QVector<EventList *> >::iterator el=(*s)->eventlist.find(code);
                     if (el==(*s)->eventlist.end()) continue;
                     for (int j=0;j<el.value().size();j++) {
                         EventList & e=*el.value()[j];
                         for (unsigned k=0;k<e.count();k++) {
                             array.push_back(e.data(k));
                         }
+                    }*/
                 }
             }
         }
-
-            //if (!open)
-                //day->CloseEvents();
-        }
         date=date.addDays(1);
     } while (date<end);
+//    for (QHash<EventStoreType, EventStoreType>::iterator k=summary.begin();k!=summary.end();k++) {
+//        for (int i=0;i<k.value();i++)  {
+//            array.push_back(k.key());
+//        }
+//    }
+
+    if (array.size()==0) return 0;
     qSort(array);
     int idx=array.size()*percent;
     if (idx>array.size()-1) idx=array.size()-1;

SleepLib/session.cpp

@@ -22,7 +22,7 @@ const quint16 filetype_data=1;
 
 // This is the uber important database version for SleepyHeads internal storage
 // Increment this after stuffing with Session's save & load code.
-const quint16 summary_version=7;
+const quint16 summary_version=9;
 const quint16 events_version=8;
 
 Session::Session(Machine * m,SessionID session)
@@ -145,7 +145,9 @@ bool Session::StoreSummary(QString filename)
     out << m_firstchan;
     out << m_lastchan;
 
-
+    out << m_valuesummary;
+    out << m_timesummary;
+    out << m_gain;
 
     // First output the Machine Code and type for each summary record
 /*    for (QHash<ChannelID,QVariant>::iterator i=settings.begin(); i!=settings.end(); i++) {
@@ -302,9 +304,20 @@ bool Session::LoadSummary(QString filename)
         in >> m_sph;
         in >> m_firstchan;
         in >> m_lastchan;
+
+        if (version >= 8) {
+            in >> m_valuesummary;
+            in >> m_timesummary;
+            if (version >= 9) {
+                in >> m_gain;
+            }
+        }
+
+
     }
 
     if (version<summary_version) {
+
         qDebug() << "Upgrading Summary file to version" << summary_version;
         UpdateSummaries();
         StoreSummary(filename);
@@ -532,6 +545,46 @@ bool Session::LoadEvents(QString filename)
     return true;
 }
 
+void Session::updateCountSummary(ChannelID code)
+{
+    QHash<ChannelID,QVector<EventList *> >::iterator ev=eventlist.find(code);
+    if (ev==eventlist.end()) return;
+
+    QHash<EventStoreType, EventStoreType> valsum;
+    QHash<EventStoreType, quint32> timesum;
+    QHash<EventStoreType, EventStoreType>::iterator vsi;
+    QHash<EventStoreType, quint32>::iterator tsi;
+
+    EventDataType raw,lastraw=0;
+    qint64 time,lasttime=0;
+    qint32 len;
+    for (int i=0;i<ev.value().size();i++) {
+        EventList & e=*(ev.value()[i]);
+        if (e.type()==EVL_Waveform) continue;
+
+        for (unsigned j=0;j<e.count();j++) {
+            raw=e.raw(j);
+            vsi=valsum.find(raw);
+            if (vsi==valsum.end()) valsum[raw]=1;
+            else vsi.value()++;
+
+            time=e.time(j);
+            if (lasttime>0) {
+                len=(time-lasttime) / 1000;
+                tsi=timesum.find(lastraw);
+                if (tsi==timesum.end()) timesum[raw]=len;
+                else tsi.value()+=len;
+            }
+            lastraw=raw;
+            lasttime=time;
+        }
+    }
+    if (valsum.size()==0) return;
+
+    m_valuesummary[code]=valsum;
+    m_timesummary[code]=timesum;
+}
+
 void Session::UpdateSummaries()
 {
     calcAHIGraph(this);
@@ -547,6 +600,12 @@ void Session::UpdateSummaries()
         id=c.key();
         if (schema::channel[id].type()==schema::DATA) {
             //sum(id); // avg calculates this and cnt.
+            if (c.value().size()>0) {
+                EventList * el=c.value()[0];
+                EventDataType gain=el->gain();
+                m_gain[id]=gain;
+            } else m_gain[id]=0;
+            updateCountSummary(id);
             Min(id);
             Max(id);
             count(id);

SleepLib/session.h

@@ -142,6 +142,11 @@ public:
     QHash<ChannelID,quint64> m_firstchan;
     QHash<ChannelID,quint64> m_lastchan;
 
+    QHash<ChannelID,QHash<EventStoreType, EventStoreType> > m_valuesummary;
+    QHash<ChannelID,QHash<EventStoreType, quint32> > m_timesummary;
+    QHash<ChannelID,EventDataType> m_gain;
+
+    void updateCountSummary(ChannelID code);
 
     // UpdateSummaries may recalculate all these, but it may be faster setting upfront
     void setCount(ChannelID id,int val) { m_cnt[id]=val; }

mainwindow.cpp

@@ -512,14 +512,13 @@ void MainWindow::on_summaryButton_clicked()
     mach.append(cpap_machines);
     mach.append(oximeters);
 
+
     int cpapdays=PROFILE.countDays(MT_CPAP,firstcpap,lastcpap);
     CPAPMode cpapmode=(CPAPMode)p_profile->calcSettingsMax(CPAP_Mode,MT_CPAP,firstcpap,lastcpap);
 
-    if (mach.size()==0) {
+    if (cpapdays==0)  {
         html+="<p>No Machine Data Imported</p>";
     } else {
-
-
         html+="<div align=center>";
         html+=QString("<p><b>Summary Information as of %1</b></p>").arg(lastcpap.toString(Qt::SystemLocaleLongDate));
         html+=QString("<table cellpadding=2 cellspacing=0 border=1 width=90%>");