Refactor main server loop a bit, to add basic retry w/ error logging

hopefully this lets us get past a simple Postgres restart..

rattail_tempmon/client.py

@@ -92,6 +92,7 @@ class TempmonClient(Daemon):
                 try:
                     session.commit()
                 except:
+                    # TODO: pretty sure we need to add a retry for this..
                     log.exception("failed to commit changes to database")
                     raise
 

rattail_tempmon/server.py

@@ -61,22 +61,28 @@ class TempmonServerDaemon(Daemon):
         self.now = make_utc()
         session = TempmonSession()
 
-        clients = session.query(tempmon.Client)\
-                         .filter(tempmon.Client.enabled == True)
-        for client in clients:
-            self.check_readings_for_client(session, client)
-
-        session.commit()
-        session.close()
-
-        # TODO: not sure this is really necessary?
-        self.set_last_checked()
+        try:
+            clients = session.query(tempmon.Client)\
+                             .filter(tempmon.Client.enabled == True)
+            for client in clients:
+                self.check_readings_for_client(session, client)
+        except:
+            log.exception("Failed to check client probe readings (but will keep trying)")
+            session.rollback()
+        else:
+            session.commit()
+        finally:
+            session.close()
 
     def check_readings_for_client(self, session, client):
         delay = client.delay or 60
         cutoff = self.now - datetime.timedelta(seconds=delay + 60)
+        online = False
         for probe in client.enabled_probes():
-            self.check_readings_for_probe(session, probe, cutoff)
+            online = bool(self.check_readings_for_probe(session, probe, cutoff))
+        if not online and client.online:
+            log.info("marking client as OFFLINE: {}".format(client))
+            client.online = False
 
     def check_readings_for_probe(self, session, probe, cutoff):
         readings = session.query(tempmon.Reading)\
@@ -103,6 +109,8 @@ class TempmonServerDaemon(Daemon):
         else: # no readings for probe
             self.update_status(probe, self.enum.TEMPMON_PROBE_STATUS_ERROR)
 
+        return readings
+
     def update_status(self, probe, status, reading=None):
         data = {
             'probe': probe,
@@ -154,12 +162,6 @@ class TempmonServerDaemon(Daemon):
 
         probe.status_alert_sent = self.now
 
-    def set_last_checked(self):
-        session = Session()
-        api.save_setting(session, 'tempmon.server.last_checked', self.now.strftime(self.timefmt))
-        session.commit()
-        session.close()
-
 
 def make_daemon(config, pidfile=None):
     """