Prevent 0x31 Pod faults and improve recovery from uncertain delivery statuses

omnipod/src/main/java/info/nightscout/androidaps/plugins/pump/omnipod/OmnipodPumpPlugin.java

@@ -506,15 +506,19 @@ public class OmnipodPumpPlugin extends PumpPluginBase implements PumpInterface,
     }
 
     /**
-     * The only actual status requests we send to the Pod here are on startup (in {@link #initializeAfterRileyLinkConnection() initializeAfterRileyLinkConnection()})
-     * And when the user explicitly requested it by clicking the Refresh button on the Omnipod tab (which is executed through {@link #executeCustomCommand(CustomCommand)})
      * We don't do periodical status requests because that could drain the Pod's battery
+     * The only actual status requests we send to the Pod here are on startup (in {@link #initializeAfterRileyLinkConnection() initializeAfterRileyLinkConnection()})
+     * And when the basal and/or temp basal status is uncertain
+     * When the user explicitly requested it by clicking the Refresh button on the Omnipod tab (which is executed through {@link #executeCustomCommand(CustomCommand)})
      */
     @Override
     public void getPumpStatus() {
         if (firstRun) {
             initializeAfterRileyLinkConnection();
             firstRun = false;
+        } else if (!podStateManager.isBasalCertain() || !podStateManager.isTempBasalCertain()) {
+            aapsLogger.info(LTag.PUMP, "Acknowledged AAPS getPumpStatus request because basal and/or temp basal is uncertain");
+            executeCommand(OmnipodCommandType.GET_POD_STATUS, aapsOmnipodManager::getPodStatus);
         }
     }
 

omnipod/src/main/java/info/nightscout/androidaps/plugins/pump/omnipod/driver/manager/OmnipodManager.java

@@ -171,6 +171,15 @@ public class OmnipodManager {
     public synchronized void setBasalSchedule(BasalSchedule schedule, boolean acknowledgementBeep) {
         assertReadyForDelivery();
 
+        if (!podStateManager.isBasalCertain()) {
+            try {
+                getPodStatus();
+            } catch (OmnipodException ex) {
+                ex.setCertainFailure(true);
+                throw ex;
+            }
+        }
+
         boolean wasSuspended = podStateManager.isSuspended();
         if (!wasSuspended) {
             try {
@@ -185,12 +194,17 @@ public class OmnipodManager {
             }
         }
 
+        BasalSchedule oldBasalSchedule = podStateManager.getBasalSchedule();
+
         try {
+            podStateManager.setBasalSchedule(schedule);
+            podStateManager.setBasalCertain(false);
             executeAndVerify(() -> communicationService.executeAction(new SetBasalScheduleAction(podStateManager, schedule,
                     false, podStateManager.getScheduleOffset(), acknowledgementBeep)));
-            podStateManager.setBasalSchedule(schedule);
         } catch (OmnipodException ex) {
             if (ex.isCertainFailure()) {
+                podStateManager.setBasalSchedule(oldBasalSchedule);
+                podStateManager.setBasalCertain(true);
                 if (!wasSuspended) {
                     throw new CommandFailedAfterChangingDeliveryStatusException("Suspending delivery succeeded but setting the new basal schedule did not", ex);
                 }
@@ -206,6 +220,19 @@ public class OmnipodManager {
     public synchronized void setTemporaryBasal(double rate, Duration duration, boolean acknowledgementBeep, boolean completionBeep) {
         assertReadyForDelivery();
 
+        if (!podStateManager.isTempBasalCertain() || !podStateManager.isBasalCertain()) {
+            try {
+                getPodStatus();
+            } catch (OmnipodException ex) {
+                ex.setCertainFailure(true);
+                throw ex;
+            }
+        }
+
+        if (podStateManager.isSuspended()) {
+            throw new IllegalDeliveryStatusException(DeliveryStatus.NORMAL, DeliveryStatus.SUSPENDED);
+        }
+
         boolean cancelCurrentTbr = podStateManager.isTempBasalRunning();
 
         if (cancelCurrentTbr) {
@@ -217,17 +244,20 @@ public class OmnipodManager {
                 }
 
                 // Uncertain failure
-                podStateManager.setTempBasalCertain(false);
                 throw new PrecedingCommandFailedUncertainlyException(ex);
             }
         }
 
         try {
+            podStateManager.setTempBasal(DateTime.now().minus(OmnipodConstants.AVERAGE_TEMP_BASAL_COMMAND_COMMUNICATION_DURATION), rate, duration);
+            podStateManager.setTempBasalCertain(false);
             executeAndVerify(() -> communicationService.executeAction(new SetTempBasalAction(
                     podStateManager, rate, duration, acknowledgementBeep, completionBeep)));
-            podStateManager.setTempBasal(DateTime.now().minus(OmnipodConstants.AVERAGE_TEMP_BASAL_COMMAND_COMMUNICATION_DURATION), rate, duration, true);
+            podStateManager.setTempBasal(DateTime.now().minus(OmnipodConstants.AVERAGE_TEMP_BASAL_COMMAND_COMMUNICATION_DURATION), rate, duration);
         } catch (OmnipodException ex) {
             if (ex.isCertainFailure()) {
+                podStateManager.clearTempBasal();
+                podStateManager.setTempBasalCertain(true);
                 if (cancelCurrentTbr) {
                     throw new CommandFailedAfterChangingDeliveryStatusException("Failed to set new TBR while cancelling old TBR succeeded", ex);
                 }
@@ -235,7 +265,6 @@ public class OmnipodManager {
             }
 
             // Uncertain failure
-            podStateManager.setTempBasal(DateTime.now().minus(OmnipodConstants.AVERAGE_TEMP_BASAL_COMMAND_COMMUNICATION_DURATION), rate, duration, false);
             throw ex;
         }
     }
@@ -247,11 +276,33 @@ public class OmnipodManager {
     private synchronized StatusResponse cancelDelivery(EnumSet<DeliveryType> deliveryTypes, boolean acknowledgementBeep) {
         assertReadyForDelivery();
 
-        return executeAndVerify(() -> {
+        if (deliveryTypes.contains(DeliveryType.BASAL)) {
-            StatusResponse statusResponse = communicationService.executeAction(new CancelDeliveryAction(podStateManager, deliveryTypes, acknowledgementBeep));
+            podStateManager.setBasalCertain(false);
-            aapsLogger.info(LTag.PUMPCOMM, "Status response after cancel delivery[types={}]: {}", deliveryTypes.toString(), statusResponse.toString());
+        }
-            return statusResponse;
+        if (deliveryTypes.contains(DeliveryType.TEMP_BASAL)) {
-        });
+            podStateManager.setTempBasalCertain(false);
+        }
+
+        try {
+            return executeAndVerify(() -> {
+                StatusResponse statusResponse;
+                statusResponse = communicationService.executeAction(new CancelDeliveryAction(podStateManager, deliveryTypes, acknowledgementBeep));
+
+                aapsLogger.info(LTag.PUMPCOMM, "Status response after cancel delivery[types={}]: {}", deliveryTypes.toString(), statusResponse.toString());
+                return statusResponse;
+            });
+        } catch (OmnipodException ex) {
+            if (ex.isCertainFailure()) {
+                if (deliveryTypes.contains(DeliveryType.BASAL)) {
+                    podStateManager.setBasalCertain(true);
+                }
+                if (deliveryTypes.contains(DeliveryType.TEMP_BASAL)) {
+                    podStateManager.setTempBasalCertain(true);
+                }
+            }
+
+            throw ex;
+        }
     }
 
     // Returns a SingleSubject that returns when the bolus has finished.
@@ -260,6 +311,19 @@ public class OmnipodManager {
     public synchronized BolusCommandResult bolus(Double units, boolean acknowledgementBeep, boolean completionBeep, BiConsumer<Double, Integer> progressIndicationConsumer) {
         assertReadyForDelivery();
 
+        if (!podStateManager.isBasalCertain()) {
+            try {
+                getPodStatus();
+            } catch (OmnipodException ex) {
+                ex.setCertainFailure(true);
+                throw ex;
+            }
+        }
+
+        if (podStateManager.isSuspended()) {
+            throw new IllegalDeliveryStatusException(DeliveryStatus.NORMAL, DeliveryStatus.SUSPENDED);
+        }
+
         bolusCommandExecutionSubject = SingleSubject.create();
 
         CommandDeliveryStatus commandDeliveryStatus = CommandDeliveryStatus.SUCCESS;

omnipod/src/main/java/info/nightscout/androidaps/plugins/pump/omnipod/driver/manager/PodStateManager.java

@@ -368,6 +368,14 @@ public abstract class PodStateManager {
         setAndStore(() -> podState.setBasalSchedule(basalSchedule));
     }
 
+    public final boolean isBasalCertain() {
+        return getSafe(() -> podState.isBasalCertain());
+    }
+
+    public final void setBasalCertain(boolean certain) {
+        setAndStore(() -> podState.setBasalCertain(certain));
+    }
+
     public final DateTime getLastBolusStartTime() {
         return getSafe(() -> podState.getLastBolusStartTime());
     }
@@ -419,11 +427,11 @@ public abstract class PodStateManager {
         setSafe(() -> podState.setTempBasalCertain(certain));
     }
 
-    public final void setTempBasal(DateTime startTime, Double amount, Duration duration, boolean certain) {
+    public final void setTempBasal(DateTime startTime, Double amount, Duration duration) {
-        setTempBasal(startTime, amount, duration, certain, true);
+        setTempBasal(startTime, amount, duration, true);
     }
 
-    public final void setTempBasal(DateTime startTime, Double amount, Duration duration, Boolean certain, boolean store) {
+    private void setTempBasal(DateTime startTime, Double amount, Duration duration, boolean store) {
         DateTime currentStartTime = getTempBasalStartTime();
         Double currentAmount = getTempBasalAmount();
         Duration currentDuration = getTempBasalDuration();
@@ -432,7 +440,6 @@ public abstract class PodStateManager {
                 podState.setTempBasalStartTime(startTime);
                 podState.setTempBasalAmount(amount);
                 podState.setTempBasalDuration(duration);
-                podState.setTempBasalCertain(certain);
             };
 
             if (store) {
@@ -444,6 +451,14 @@ public abstract class PodStateManager {
         }
     }
 
+    public final void clearTempBasal() {
+        clearTempBasal(true);
+    }
+
+    private void clearTempBasal(boolean store) {
+        setTempBasal(null, null, null, store);
+    }
+
     /**
      * @return true when a Temp Basal is stored in the Pod Stated
      * Please note that this could also be an expired Temp Basal. For an indication on whether or not
@@ -457,13 +472,22 @@ public abstract class PodStateManager {
      * @return true when a Temp Basal is stored in the Pod State and this temp basal is currently running (based on start time and duration)
      */
     public final boolean isTempBasalRunning() {
-        return isTempBasalRunningAt(DateTime.now());
+        return isTempBasalRunningAt(null);
     }
 
     /**
-     * @return true when a Temp Basal is stored in the Pod State and this temp basal is running at the given time (based on start time and duration)
+     * @param time the time for which to look up whether a temp basal is running, null meaning now
+     * @return true when a Temp Basal is stored in the Pod State and this temp basal is running at the given time (based on start time and duration),
+     * or when the time provided is null and the delivery status of the Pod inidicated that a TBR is running, but not TBR is stored
+     * This can happen in some rare cases.
      */
     public final boolean isTempBasalRunningAt(DateTime time) {
+        if (time == null) { // now
+            if (!hasTempBasal() && getLastDeliveryStatus().isTbrRunning()) {
+                return true;
+            }
+            time = DateTime.now();
+        }
         if (hasTempBasal()) {
             DateTime tempBasalStartTime = getTempBasalStartTime();
             DateTime tempBasalEndTime = tempBasalStartTime.plus(getTempBasalDuration());
@@ -537,15 +561,12 @@ public abstract class PodStateManager {
             podState.setTotalTicksDelivered(status.getTicksDelivered());
             podState.setPodProgressStatus(status.getPodProgressStatus());
             podState.setTimeActive(status.getTimeActive());
-            if (status.getDeliveryStatus().isTbrRunning()) {
+            if (!status.getDeliveryStatus().isTbrRunning()) {
-                if (!isTempBasalCertain() && isTempBasalRunning()) {
+                clearTempBasal(false);
-                    podState.setTempBasalCertain(true);
-                }
-            } else {
-                // Triggers {@link #onTbrChanged() onTbrChanged()} when appropriate
-                setTempBasal(null, null, null, true, false);
             }
             podState.setLastUpdatedFromResponse(DateTime.now());
+            podState.setTempBasalCertain(true);
+            podState.setBasalCertain(true);
 
             if (status instanceof PodInfoDetailedStatus) {
                 PodInfoDetailedStatus detailedStatus = (PodInfoDetailedStatus) status;
@@ -667,6 +688,7 @@ public abstract class PodStateManager {
         private DeliveryStatus lastDeliveryStatus;
         private AlertSet activeAlerts;
         private BasalSchedule basalSchedule;
+        private boolean basalCertain;
         private DateTime lastBolusStartTime;
         private Double lastBolusAmount;
         private Duration lastBolusDuration;
@@ -871,6 +893,14 @@ public abstract class PodStateManager {
             this.basalSchedule = basalSchedule;
         }
 
+        Boolean isBasalCertain() {
+            return basalCertain;
+        }
+
+        void setBasalCertain(Boolean certain) {
+            this.basalCertain = certain;
+        }
+
         DateTime getLastBolusStartTime() {
             return lastBolusStartTime;
         }

omnipod/src/main/java/info/nightscout/androidaps/plugins/pump/omnipod/ui/OmnipodOverviewFragment.kt

@@ -361,11 +361,17 @@ class OmnipodOverviewFragment : DaggerFragment() {
             }
         } else {
             if (podStateManager.podProgressStatus.isRunning) {
-                if (podStateManager.isSuspended) {
+                var status = if (podStateManager.isSuspended) {
                     resourceHelper.gs(R.string.omnipod_pod_status_suspended)
                 } else {
                     resourceHelper.gs(R.string.omnipod_pod_status_running)
                 }
+
+                if (!podStateManager.isBasalCertain) {
+                    status += " (" + resourceHelper.gs(R.string.omnipod_uncertain) + ")"
+                }
+
+                status
             } else if (podStateManager.podProgressStatus == PodProgressStatus.FAULT_EVENT_OCCURRED) {
                 resourceHelper.gs(R.string.omnipod_pod_status_pod_fault)
             } else if (podStateManager.podProgressStatus == PodProgressStatus.INACTIVE) {
@@ -375,7 +381,7 @@ class OmnipodOverviewFragment : DaggerFragment() {
             }
         }
 
-        val podStatusColor = if (!podStateManager.isPodActivationCompleted || podStateManager.isPodDead || podStateManager.isSuspended) {
+        val podStatusColor = if (!podStateManager.isPodActivationCompleted || podStateManager.isPodDead || podStateManager.isSuspended || (podStateManager.isPodRunning && !podStateManager.isBasalCertain)) {
             Color.RED
         } else {
             Color.WHITE
@@ -406,26 +412,31 @@ class OmnipodOverviewFragment : DaggerFragment() {
 
     private fun updateTempBasal() {
         if (podStateManager.isPodActivationCompleted && podStateManager.isTempBasalRunning) {
-            val now = DateTime.now()
+            if (!podStateManager.hasTempBasal()) {
-
+                omnipod_overview_temp_basal.text = "???"
-            val startTime = podStateManager.tempBasalStartTime
+                omnipod_overview_temp_basal.setTextColor(Color.RED)
-            val amount = podStateManager.tempBasalAmount
-            val duration = podStateManager.tempBasalDuration
-
-            val minutesRunning = Duration(startTime, now).standardMinutes
-
-            var text: String
-            val textColor: Int
-            text = resourceHelper.gs(R.string.omnipod_overview_temp_basal_value, amount, dateUtil.timeString(startTime.millis), minutesRunning, duration.standardMinutes)
-            if (podStateManager.isTempBasalCertain) {
-                textColor = Color.WHITE
             } else {
-                textColor = Color.RED
+                val now = DateTime.now()
-                text += " (" + resourceHelper.gs(R.string.omnipod_uncertain) + ")"
-            }
 
-            omnipod_overview_temp_basal.text = text
+                val startTime = podStateManager.tempBasalStartTime
-            omnipod_overview_temp_basal.setTextColor(textColor)
+                val amount = podStateManager.tempBasalAmount
+                val duration = podStateManager.tempBasalDuration
+
+                val minutesRunning = Duration(startTime, now).standardMinutes
+
+                var text: String
+                val textColor: Int
+                text = resourceHelper.gs(R.string.omnipod_overview_temp_basal_value, amount, dateUtil.timeString(startTime.millis), minutesRunning, duration.standardMinutes)
+                if (podStateManager.isTempBasalCertain) {
+                    textColor = Color.WHITE
+                } else {
+                    textColor = Color.RED
+                    text += " (" + resourceHelper.gs(R.string.omnipod_uncertain) + ")"
+                }
+
+                omnipod_overview_temp_basal.text = text
+                omnipod_overview_temp_basal.setTextColor(textColor)
+            }
         } else {
             omnipod_overview_temp_basal.text = PLACEHOLDER
             omnipod_overview_temp_basal.setTextColor(Color.WHITE)