Add watchdog to avoid endless hanging rounds

3 files changed, 72 insertions, 9 deletions

diff --git a/src/CRSMConfig.hpp b/src/CRSMConfig.hpp
index f3cca99..dd59259 100644
--- a/src/CRSMConfig.hpp
+++ b/src/CRSMConfig.hpp
@@ -31,6 +31,11 @@ public:
             List(String) IgnoreFolders = {"Network", "Records.c4f", "Savegames.c4f"};
 
             Integer EmptyTimer = 60;
+
+            struct {
+                Integer Interval = 10;
+                Integer Timeout = 60;
+            } Watchdog;
         } Server;
 
         struct {
@@ -133,6 +138,9 @@ public:
         ConfigVal(Clonk.Server.EmptyTimer),
         ConfigVal(Clonk.Server.IgnoreFolders),
 
+        ConfigVal(Clonk.Server.Watchdog.Interval),
+        ConfigVal(Clonk.Server.Watchdog.Timeout),
+
 
         ConfigVal(Clonk.Chat.AntiFlood.Count),
         ConfigVal(Clonk.Chat.AntiFlood.Time),
diff --git a/src/crsm.cpp b/src/crsm.cpp
index 636cd0e..8c9e371 100644
--- a/src/crsm.cpp
+++ b/src/crsm.cpp
@@ -61,6 +61,10 @@ CRSM::CRSM(QObject *parent) :
     {
         Session.read(Config.CRSM.SessionFile, false);
         writeToServer("Der Server Manager läuft wieder.\n");
+        if(Session.State == CRSMSession::Running)
+        {
+            watchdog();
+        }
     }
     QFile sessionFile(Config.CRSM.SessionFile);
     if(sessionFile.exists()) sessionFile.remove();
@@ -78,6 +82,8 @@ CRSM::CRSM(QObject *parent) :
     connect(&gameRegisterFailTimer, SIGNAL(timeout()), this, SLOT(enableAutoHosting()));
     gameRegisterFailTimer.setInterval(5*60*1000);
 
+    connect(&watchDogTimer, SIGNAL(timeout()), this, SLOT(watchdog()));
+
     ok = true;
 }
 
@@ -229,6 +235,16 @@ void CRSM::readServerOutput()
             return;
         }
 
+        if(type == "Wtd")
+        {
+            if(what == watchDogString)
+            {
+                watchDogString.clear();
+                watchDogTimer.start(Config.Clonk.Server.Watchdog.Interval * 1000);
+            }
+            return;
+        }
+
         ClientInfo* clientPtr = parseClientInfo(what);
         if(clientPtr == nullptr)
         {
@@ -365,6 +381,7 @@ void CRSM::readServerOutput()
             writeToServer(QString("/set maxplayer 0\n"));
         }
         setSessionState(CRSMSession::Running);
+        watchdog();
         return;
     }
 
@@ -387,15 +404,7 @@ void CRSM::readServerOutput()
             }
             autoHost = false;
             static const QString gameRegisterFailMessage = "Aufgrund eines Problems beim Registrieren des Spiels am Masterserver (%1) wird Hosting temporär (für 5 Minuten) deaktiviert.\n";
-            out(gameRegisterFailMessage.arg(reason));
-            if(Config.IRC.Use)
-            {
-                sendIrcMessage(gameRegisterFailMessage.arg(reason), Config.IRC.Channel, false, false);
-                if(Config.IRC.UseIngameChat)
-                {
-                    sendIrcMessage(gameRegisterFailMessage.arg(reason), Config.IRC.IngameChannel, false, false);
-                }
-            }
+            announceInfo(gameRegisterFailMessage.arg(reason));
             return;
         }
     }
@@ -438,6 +447,8 @@ void CRSM::scenarioFinished()
         return;
     }
     Session.clear();
+    watchDogString.clear();
+    watchDogTimer.stop();
     if((autoHost || userlist.length() > 0) && !finish)
     {
         nextScen();
@@ -1804,6 +1815,19 @@ void CRSM::replayOutputBuffer(QTcpSocket *socket, bool clear)
     }
 }
 
+void CRSM::announceInfo(const QString &info)
+{
+    out(info);
+    if(Config.IRC.Use)
+    {
+        sendIrcMessage(info, Config.IRC.Channel, false, false);
+        if(Config.IRC.UseIngameChat)
+        {
+            sendIrcMessage(info, Config.IRC.IngameChannel, false, false);
+        }
+    }
+}
+
 bool CRSM::scenAllowed(const ScenarioSettings &scen, const ClientInfo &client, UserType userType)
 {
     if(userType >= UserType::Moderator)
@@ -1921,6 +1945,16 @@ void CRSM::applyConfig()
     Config.Auto.Volatile.Clonk.Directory = QFileInfo(Config.Clonk.Server.Executable).absoluteDir().absolutePath() + QDir::separator();
     afkAdminTimer.setInterval(Config.Hosting.AfkAdminTime * 1000);
 
+    if(Config.Clonk.Server.Watchdog.Interval > 0 && Config.Clonk.Server.Watchdog.Timeout > 0 && Session.State == CRSMSession::Running)
+    {
+        watchDogString.clear();
+        watchdog();
+    }
+    else
+    {
+        watchDogTimer.stop();
+    }
+
     Config.CRSM.ListFolder = QDir(Config.CRSM.ListFolder).absolutePath() + QDir::separator();
     Config.Logging.Folder = QDir(Config.Logging.Folder).absolutePath() + QDir::separator();
 
@@ -2348,6 +2382,21 @@ void CRSM::handleIrcMessage(const ClientInfo &client, QString message, const QSt
     }
 }
 
+void CRSM::watchdog()
+{
+    if(watchDogString.isEmpty())
+    {
+        watchDogTimer.start(Config.Clonk.Server.Watchdog.Timeout * 1000);
+        watchDogString = GetRandomString(20);
+        writeToServer("/watchdog " + watchDogString + "\n");
+    }
+    else
+    {
+        announceInfo("Clonk reagiert nicht und wird beendet.\n");
+        processManager->kill();
+    }
+}
+
 CMD_FUNCTION_IMPL(help)
     bool recursive = args.startsWith("recursive ") && client.interface == Management;
     ClientInterface interface = client.interface;
diff --git a/src/crsm.hpp b/src/crsm.hpp
index 45c4aea..bac0584 100644
--- a/src/crsm.hpp
+++ b/src/crsm.hpp
@@ -146,6 +146,8 @@ private slots:
     void enableAutoHosting();
     void afkAdminTimeout();
 
+    void watchdog();
+
 private:
     CRSMConfig Config;
 
@@ -234,6 +236,8 @@ private:
     QMap<QTcpSocket*, ManagementConnection> managementConnections;
     QTimer gameRegisterFailTimer;
     QTimer afkAdminTimer;
+    QTimer watchDogTimer;
+    QString watchDogString;
 
     bool ok = false;
     bool hostingIsErrorDeactivated = false;
@@ -291,6 +295,8 @@ private:
     void out(const QString& text);
     void replayOutputBuffer(QTcpSocket *socket, bool clear = true);
 
+    void announceInfo(const QString& info);
+
     bool scenAllowed(const ScenarioSettings& scen, const ClientInfo& client, UserType userType);
 
     void kick(const QString& pcName, const QString& reason = "");