make Failed stated recoverable (#137)

A process in the failed state can transition to stopped either by calling /unload or swapping to another model.

README.md

@@ -131,6 +131,8 @@ models:
 
     # use a custom command to stop the model when swapping. By default
     # this is SIGTERM on POSIX systems, and taskkill on Windows systems
+    # the ${PID} variable can be used in cmdStop, it will be automatically replaced
+    # with the PID of the running model
     cmdStop: docker stop dockertest
 
 # Groups provide advanced controls over model swapping behaviour. Using groups

proxy/process.go

@@ -149,7 +149,9 @@ func isValidTransition(from, to ProcessState) bool {
 		return to == StateStopping
 	case StateStopping:
 		return to == StateStopped || to == StateShutdown
-	case StateFailed, StateShutdown:
+	case StateFailed:
+		return to == StateStopping
+	case StateShutdown:
 		return false // No transitions allowed from these states
 	}
 	return false
@@ -359,13 +361,20 @@ func (p *Process) StopImmediately() {
 		return
 	}
 
-	p.proxyLogger.Debugf("<%s> Stopping process", p.ID)
+	p.proxyLogger.Debugf("<%s> Stopping process, current state: %s", p.ID, p.CurrentState())
+	currentState := p.CurrentState()
 
-	// calling Stop() when state is invalid is a no-op
+	if currentState == StateFailed {
+		if curState, err := p.swapState(StateFailed, StateStopping); err != nil {
+			p.proxyLogger.Infof("<%s> Stop() Failed -> StateStopping err: %v, current state: %v", p.ID, err, curState)
+			return
+		}
+	} else {
 		if curState, err := p.swapState(StateReady, StateStopping); err != nil {
 			p.proxyLogger.Infof("<%s> Stop() Ready -> StateStopping err: %v, current state: %v", p.ID, err, curState)
 			return
 		}
+	}
 
 	// stop the process with a graceful exit timeout
 	p.stopCommand(p.gracefulStopTimeout)

proxy/proxymanager.go

@@ -352,6 +352,8 @@ func (pm *ProxyManager) upstreamIndex(c *gin.Context) {
 					stateStr = "Failed"
 				case StateShutdown:
 					stateStr = "Shutdown"
+				case StateStopped:
+					stateStr = "Stopped"
 				default:
 					stateStr = "Unknown"
 				}