Overhaul fallback logic

Fallback is now set up by configuring individual servers to build up a
fallback chain. Each server now only accepts a single fallback server
name and global fallback server definitions have been removed entirely.
Fallback is attempted if there is a kick or if the RUDP connection is
lost. If following the fallback chain is unsuccessful for any reason,
the client is disconnected with the original kick sent by the original
server or an error message about the loss of connection. If fallback
cannot be initiated at any part of the chain (e.g. due to the client not
being connected to a server in the first place) the client is
disconnected with an error message.

This commit fixes #150.

10 files changed, 116 insertions, 109 deletions

diff --git a/client_conn.go b/client_conn.go
index 951fa95..9753147 100644
--- a/client_conn.go
+++ b/client_conn.go
@@ -40,6 +40,9 @@ type ClientConn struct {
 		salt, srpA, srpB, srpM, srpK []byte
 	}
 
+	fallbackFrom string
+	whyKicked    *mt.ToCltKick
+
 	lang string
 
 	major, minor, patch uint8
diff --git a/config.go b/config.go
index 03b1eae..9b6f1e3 100644
--- a/config.go
+++ b/config.go
@@ -29,7 +29,7 @@ type Server struct {
 	Addr      string
 	MediaPool string
 	Groups    []string
-	Fallbacks []string
+	Fallback  string
 
 	dynamic   bool
 	poolAdded time.Time
@@ -53,7 +53,6 @@ type Config struct {
 	Servers          map[string]Server
 	ForceDefaultSrv  bool
 	KickOnNewPool    bool
-	FallbackServers  []string
 	CSMRF            struct {
 		NoCSMs          bool
 		ChatMsgs        bool
@@ -165,9 +164,6 @@ func (cnf Config) clone() Config {
 
 	newConfig.Servers = copyMap(cnf.Servers)
 
-	newConfig.FallbackServers = make([]string, len(cnf.FallbackServers))
-	copy(newConfig.FallbackServers, cnf.FallbackServers)
-
 	newConfig.Groups = copyMapSlice(cnf.Groups)
 	newConfig.UserGroups = copyMap(cnf.UserGroups)
 
@@ -286,29 +282,6 @@ func (cnf Config) RandomGroupServer(search string) (string, bool) {
 	return candidates[rand.Intn(len(candidates))], true
 }
 
-// FallbackServers returns a slice of server names that
-// a server can fall back to.
-func FallbackServers(server string) []string {
-	conf := Conf()
-
-	srv, ok := conf.Servers[server]
-	if !ok {
-		return nil
-	}
-
-	fallbacks := srv.Fallbacks
-	fallbacks = append(fallbacks, conf.FallbackServers...)
-
-	final := make([]string, 0, len(fallbacks))
-	for _, srvName := range fallbacks {
-		if srvName != server {
-			final = append(final, srvName)
-		}
-	}
-
-	return final
-}
-
 // LoadConfig attempts to parse the configuration file.
 // It leaves the config unchanged if there is an error
 // and returns the error.
@@ -325,7 +298,6 @@ func LoadConfig() error {
 	config.TelnetAddr = defaultTelnetAddr
 	config.BindAddr = defaultBindAddr
 	config.Servers = make(map[string]Server)
-	config.FallbackServers = make([]string, 0)
 	config.Groups = make(map[string][]string)
 	config.UserGroups = make(map[string]string)
 	config.List.Interval = defaultListInterval
diff --git a/connect.go b/connect.go
index 743f00d..632d203 100644
--- a/connect.go
+++ b/connect.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"time"
 
 	"github.com/HimbeerserverDE/mt"
 )
@@ -42,8 +43,18 @@ func connect(conn net.Conn, name string, cc *ClientConn) *ServerConn {
 
 	cc.mu.Lock()
 	cc.srv = sc
+	cc.fallbackFrom = sc.name
 	cc.mu.Unlock()
 
+	// Mark fallback as done after a connection has persisted for some time.
+	go func() {
+		time.Sleep(10 * time.Second)
+
+		if cc.srv == sc {
+			cc.whyKicked = nil
+		}
+	}()
+
 	go handleSrv(sc)
 	return sc
 }
diff --git a/doc/config.md b/doc/config.md
index 18a4006..e9e4e01 100644
--- a/doc/config.md
+++ b/doc/config.md
@@ -140,15 +140,13 @@ See [server_groups.md](https://github.com/HimbeerserverDE/mt-multiserver-proxy/b
 for more information.
 ```
 
-> `Server.Fallbacks`
+> `Server.Fallback`
 ```
-Type: []string
-Default: []string{}
-Description: The names of the servers a client should fall back to
-if this server shuts down or crashes gracefully. Connection attempts
-are made in the order in which the servers are given. As soon as
-a connection is successful the other fallback servers in this list
-will be ignored.
+Type: string
+Default: ""
+Description: The name of the server a client should fall back to if this server
+shuts down or crashes gracefully. Connection attempts are made in the order in
+which the servers are given.
 ```
 
 > `ForceDefaultSrv`
@@ -224,15 +222,6 @@ Default: 0
 Description: The maximum distance from which CSMs can read the map.
 ```
 
-> `FallbackServers`
-```
-Type: []string
-Default: []string{}
-Description: Names of general fallback servers to connect to
-if a connection attempt fails or an existing connection
-to a game server is lost.
-```
-
 > `DropCSMRF`
 ```
 Type: bool
diff --git a/doc/server_groups.md b/doc/server_groups.md
index 4a0140a..14b9b86 100644
--- a/doc/server_groups.md
+++ b/doc/server_groups.md
@@ -5,7 +5,7 @@ in the `Groups` subfield of the server definition in the config.
 Configuration options that support server groups will randomly choose
 from their member servers every time they are applied to a client.
 
-Neither local nor global fallback servers can be server groups.
+Fallback servers cannot be server groups.
 
 If there is a server group with the same name as a regular server,
 the regular server is preferred, rendering the group inaccessible.
diff --git a/fallback.go b/fallback.go
new file mode 100644
index 0000000..729a4e4
--- /dev/null
+++ b/fallback.go
@@ -0,0 +1,61 @@
+package proxy
+
+import "github.com/HimbeerserverDE/mt"
+
+func (cc *ClientConn) fallback() bool {
+	if cc.fallbackFrom == "" {
+		return false
+	}
+
+	fallback := config.Servers[cc.fallbackFrom].Fallback
+	if fallback == "" {
+		ack, _ := cc.SendCmd(cc.whyKicked)
+
+		select {
+		case <-cc.Closed():
+		case <-ack:
+			cc.Close()
+
+			cc.mu.Lock()
+			cc.srv = nil
+			cc.mu.Unlock()
+
+			if srv := cc.server(); srv != nil {
+				srv.mu.Lock()
+				srv.clt = nil
+				srv.mu.Unlock()
+			}
+		}
+
+		return true
+	}
+
+	// Use HopRaw so the fallback server doesn't get saved
+	// as the last server.
+	if err := cc.HopRaw(fallback); err != nil {
+		cc.Log("<-", "fallback fail:", err)
+
+		ack, _ := cc.SendCmd(&mt.ToCltKick{
+			Reason: mt.Custom,
+			Custom: "Fallback failed.",
+		})
+
+		select {
+		case <-cc.Closed():
+		case <-ack:
+			cc.Close()
+
+			cc.mu.Lock()
+			cc.srv = nil
+			cc.mu.Unlock()
+
+			if srv := cc.server(); srv != nil {
+				srv.mu.Lock()
+				srv.clt = nil
+				srv.mu.Unlock()
+			}
+		}
+	}
+
+	return true
+}
diff --git a/hop.go b/hop.go
index 4d8ac77..de1774a 100644
--- a/hop.go
+++ b/hop.go
@@ -28,27 +28,8 @@ func (cc *ClientConn) Hop(serverName string) (err error) {
 		}
 	}()
 
-	if err = cc.HopRaw(serverName); err != nil {
-		if errors.Is(err, ErrNoSuchServer) || errors.Is(err, ErrNewMediaPool) {
-			return err
-		}
-
-		cc.Log("<-", err)
-		cc.SendChatMsg("Could not switch servers, triggering fallback. Error:", err.Error())
-
-		for _, srvName := range FallbackServers(serverName) {
-			if err = cc.HopRaw(srvName); err != nil {
-				cc.Log("<-", err)
-				cc.SendChatMsg("Could not connect, continuing fallback. Error:", err.Error())
-			}
-
-			return nil
-		}
-
-		return err
-	}
-
-	return nil
+	err = cc.HopRaw(serverName)
+	return
 }
 
 // HopGroup connects the ClientConn to the specified server group
diff --git a/process.go b/process.go
index 8144062..82b0bce 100644
--- a/process.go
+++ b/process.go
@@ -568,16 +568,12 @@ func (sc *ServerConn) process(pkt mt.Pkt) {
 		sc.Log("<-", "deny access", cmd)
 
 		if cmd.Reason == mt.Shutdown || cmd.Reason == mt.Crash || cmd.Reason == mt.SrvErr || cmd.Reason == mt.TooManyClts || cmd.Reason == mt.UnsupportedVer {
-			clt.SendChatMsg("A kick occured, triggering fallback. Reason:", cmd.String())
+			clt.SendChatMsg("A kick occured, switching to fallback server. Reason:", cmd)
 
-			for _, srvName := range FallbackServers(sc.name) {
-				if err := clt.HopRaw(srvName); err != nil {
-					clt.Log("<-", err)
-					clt.SendChatMsg("Could not connect to "+srvName+", continuing fallback. Error:", err.Error())
-				}
+			clt.whyKicked = cmd
 
-				return
-			}
+			clt.fallback()
+			return
 		}
 
 		ack, _ := clt.SendCmd(cmd)
diff --git a/run.go b/run.go
index 0e3c612..703e830 100644
--- a/run.go
+++ b/run.go
@@ -156,18 +156,24 @@ func runFunc() {
 
 			if err := doConnect(srvName, srv); err != nil {
 				cc.Log("<-", err)
-				cc.SendChatMsg("Could not connect, triggering fallback. Error:", err.Error())
+				cc.SendChatMsg("Could not connect, trying fallback server. Error:", err)
 
-				for _, fbName := range FallbackServers(srvName) {
-					fb, ok := conf.Servers[fbName]
+				fbName := srv.Fallback
+				for fbName != "" {
+					var ok bool
+					srv, ok = conf.Servers[fbName]
 					if !ok {
 						cc.Log("<-", "invalid fallback")
 						continue
 					}
 
-					if err := doConnect(fbName, fb); err != nil {
+					if err := doConnect(fbName, srv); err != nil {
+						fbName = srv.Fallback
+
 						cc.Log("<-", err)
-						cc.SendChatMsg("Could not connect, continuing fallback. Error:", err.Error())
+						cc.SendChatMsg("Could not connect, trying next fallback server. Error:", err.Error())
+					} else {
+						break
 					}
 				}
 			}
diff --git a/server_conn.go b/server_conn.go
index ec3164f..f451cd5 100644
--- a/server_conn.go
+++ b/server_conn.go
@@ -104,7 +104,6 @@ func handleSrv(sc *ServerConn) {
 		}
 	}()
 
-RecvLoop:
 	for {
 		pkt, err := sc.Recv()
 		if err != nil {
@@ -117,38 +116,27 @@ RecvLoop:
 
 				if sc.client() != nil {
 					if errors.Is(sc.WhyClosed(), rudp.ErrTimedOut) {
-						sc.client().SendChatMsg("Server connection timed out, triggering fallback.")
-					} else {
-						sc.client().SendChatMsg("Server connection lost, triggering fallback.")
-					}
+						sc.client().SendChatMsg("Server connection timed out, switching to fallback server.")
 
-					for _, srvName := range FallbackServers(sc.name) {
-						if err := sc.client().HopRaw(srvName); err != nil {
-							sc.client().Log("<-", err)
-							sc.client().SendChatMsg("Could not connect to "+srvName+", continuing fallback. Error:", err.Error())
+						if sc.client().whyKicked == nil {
+							sc.client().whyKicked = &mt.ToCltKick{
+								Reason: mt.Custom,
+								Custom: "Server connection timed out.",
+							}
 						}
+					} else {
+						sc.client().SendChatMsg("Server connection lost, switching to fallback server.")
 
-						break RecvLoop
+						if sc.client().whyKicked == nil {
+							sc.client().whyKicked = &mt.ToCltKick{
+								Reason: mt.Custom,
+								Custom: "Server connection lost.",
+							}
+						}
 					}
 
-					ack, _ := sc.client().SendCmd(&mt.ToCltKick{
-						Reason: mt.Custom,
-						Custom: "Server connection closed unexpectedly.",
-					})
-
-					select {
-					case <-sc.client().Closed():
-					case <-ack:
-						sc.client().Close()
-
-						sc.client().mu.Lock()
-						sc.client().srv = nil
-						sc.client().mu.Unlock()
-
-						sc.mu.Lock()
-						sc.clt = nil
-						sc.mu.Unlock()
-					}
+					sc.client().fallback()
+					break
 				}
 
 				break