Stability improvements made by reverting channelized packet distribution to queue based.

Author: IIamaseconds

go.mod

@@ -7,32 +7,25 @@ require (
 	github.com/joho/godotenv v1.5.1
 	github.com/pion/dtls/v3 v3.0.7
 	github.com/pion/ice/v4 v4.0.10
-	github.com/pion/interceptor v0.1.41
+	github.com/pion/interceptor v0.1.42
 	github.com/pion/rtcp v1.2.16
-	github.com/pion/rtp v1.8.24
+	github.com/pion/rtp v1.8.25
 	github.com/pion/webrtc/v4 v4.1.6
 )
 
-require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
-)
-
 require (
 	github.com/pion/datachannel v1.5.10 // indirect
 	github.com/pion/logging v0.2.4 // indirect
-	github.com/pion/mdns/v2 v2.0.7 // indirect
+	github.com/pion/mdns/v2 v2.1.0 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/sctp v1.8.40 // indirect
 	github.com/pion/sdp/v3 v3.0.16
 	github.com/pion/srtp/v3 v3.0.8 // indirect
-	github.com/pion/stun/v3 v3.0.0 // indirect
-	github.com/pion/transport/v3 v3.0.8 // indirect
-	github.com/pion/turn/v4 v4.1.1 // indirect
-	github.com/stretchr/testify v1.11.1
+	github.com/pion/stun/v3 v3.0.1 // indirect
+	github.com/pion/transport/v3 v3.1.1 // indirect
+	github.com/pion/turn/v4 v4.1.3 // indirect
 	github.com/wlynxg/anet v0.0.5 // indirect
-	golang.org/x/crypto v0.43.0 // indirect
-	golang.org/x/net v0.46.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/crypto v0.44.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
 )

go.sum

@@ -4,40 +4,60 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o=
 github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M=
 github.com/pion/dtls/v3 v3.0.7 h1:bItXtTYYhZwkPFk4t1n3Kkf5TDrfj6+4wG+CZR8uI9Q=
 github.com/pion/dtls/v3 v3.0.7/go.mod h1:uDlH5VPrgOQIw59irKYkMudSFprY9IEFCqz/eTz16f8=
 github.com/pion/ice/v4 v4.0.10 h1:P59w1iauC/wPk9PdY8Vjl4fOFL5B+USq1+xbDcN6gT4=
 github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw=
+github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4=
+github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic=
 github.com/pion/interceptor v0.1.41 h1:NpvX3HgWIukTf2yTBVjVGFXtpSpWgXjqz7IIpu7NsOw=
 github.com/pion/interceptor v0.1.41/go.mod h1:nEt4187unvRXJFyjiw00GKo+kIuXMWQI9K89fsosDLY=
+github.com/pion/interceptor v0.1.42 h1:0/4tvNtruXflBxLfApMVoMubUMik57VZ+94U0J7cmkQ=
+github.com/pion/interceptor v0.1.42/go.mod h1:g6XYTChs9XyolIQFhRHOOUS+bGVGLRfgTCUzH29EfVU=
 github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8=
 github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so=
 github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
 github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
+github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY=
+github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A=
 github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
+github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo=
+github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0=
 github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo=
 github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo=
-github.com/pion/rtp v1.8.24 h1:+ICyZXUQDv95EsHN70RrA4XKJf5MGWyC6QQc1u6/ynI=
-github.com/pion/rtp v1.8.24/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
+github.com/pion/rtp v1.8.22 h1:8NCVDDF+uSJmMUkjLJVnIr/HX7gPesyMV1xFt5xozXc=
+github.com/pion/rtp v1.8.22/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
+github.com/pion/rtp v1.8.25 h1:b8+y44GNbwOJTYWuVan7SglX/hMlicVCAtL50ztyZHw=
+github.com/pion/rtp v1.8.25/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
+github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE=
+github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE=
 github.com/pion/sctp v1.8.40 h1:bqbgWYOrUhsYItEnRObUYZuzvOMsVplS3oNgzedBlG8=
 github.com/pion/sctp v1.8.40/go.mod h1:SPBBUENXE6ThkEksN5ZavfAhFYll+h+66ZiG6IZQuzo=
 github.com/pion/sdp/v3 v3.0.16 h1:0dKzYO6gTAvuLaAKQkC02eCPjMIi4NuAr/ibAwrGDCo=
 github.com/pion/sdp/v3 v3.0.16/go.mod h1:9tyKzznud3qiweZcD86kS0ff1pGYB3VX+Bcsmkx6IXo=
+github.com/pion/srtp/v3 v3.0.7 h1:QUElw0A/FUg3MP8/KNMZB3i0m8F9XeMnTum86F7S4bs=
+github.com/pion/srtp/v3 v3.0.7/go.mod h1:qvnHeqbhT7kDdB+OGB05KA/P067G3mm7XBfLaLiaNF0=
 github.com/pion/srtp/v3 v3.0.8 h1:RjRrjcIeQsilPzxvdaElN0CpuQZdMvcl9VZ5UY9suUM=
 github.com/pion/srtp/v3 v3.0.8/go.mod h1:2Sq6YnDH7/UDCvkSoHSDNDeyBcFgWL0sAVycVbAsXFg=
 github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
 github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
+github.com/pion/stun/v3 v3.0.1 h1:jx1uUq6BdPihF0yF33Jj2mh+C9p0atY94IkdnW174kA=
+github.com/pion/stun/v3 v3.0.1/go.mod h1:RHnvlKFg+qHgoKIqtQWMOJF52wsImCAf/Jh5GjX+4Tw=
+github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
+github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
 github.com/pion/transport/v3 v3.0.8 h1:oI3myyYnTKUSTthu/NZZ8eu2I5sHbxbUNNFW62olaYc=
 github.com/pion/transport/v3 v3.0.8/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
+github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM=
+github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
 github.com/pion/turn/v4 v4.1.1 h1:9UnY2HB99tpDyz3cVVZguSxcqkJ1DsTSZ+8TGruh4fc=
 github.com/pion/turn/v4 v4.1.1/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8=
+github.com/pion/turn/v4 v4.1.3 h1:jVNW0iR05AS94ysEtvzsrk3gKs9Zqxf6HmnsLfRvlzA=
+github.com/pion/turn/v4 v4.1.3/go.mod h1:TD/eiBUf5f5LwXbCJa35T7dPtTpCHRJ9oJWmyPLVT3A=
+github.com/pion/webrtc/v4 v4.1.4 h1:/gK1ACGHXQmtyVVbJFQDxNoODg4eSRiFLB7t9r9pg8M=
+github.com/pion/webrtc/v4 v4.1.4/go.mod h1:Oab9npu1iZtQRMic3K3toYq5zFPvToe/QBw7dMI2ok4=
 github.com/pion/webrtc/v4 v4.1.6 h1:srHH2HwvCGwPba25EYJgUzgLqCQoXl1VCUnrGQMSzUw=
 github.com/pion/webrtc/v4 v4.1.6/go.mod h1:wKecGRlkl3ox/As/MYghJL+b/cVXMEhoPMJWPuGQFhU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -46,14 +66,23 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
 github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
+golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
+golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
 golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
 golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
+golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
+golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
+golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
 golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
 golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
 golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/environment/variables.go

@@ -8,6 +8,7 @@ const (
 	NETWORK_TEST_ON_START              = "NETWORK_TEST_ON_START"
 	INCLUDE_PUBLIC_IP_IN_NAT_1_TO_1_IP = "INCLUDE_PUBLIC_IP_IN_NAT_1_TO_1_IP"
 	DISABLE_STATUS                     = "DISABLED_STATUS"
+	ENABLE_PROFILING                   = "ENABLE_PROFILING"
 
 	// SSL
 	USE_SSL  = "USE_SSL"
@@ -59,4 +60,8 @@ const (
 	LOGGING_NEW_FILE_ON_STARTUP = "LOGGING_NEW_FILE_ON_STARTUP"
 	LOGGING_API_ENABLED         = "LOGGING_API_ENABLED"
 	LOGGING_API_KEY             = "LOGGING_API_KEY"
+
+	// WHEP SESSION
+	WHEP_SESSION_AUDIOCHANNEL_SIZE = "WHEP_SESSION_AUDIOCHANNEL_SIZE"
+	WHEP_SESSION_VIDEOCHANNEL_SIZE = "WHEP_SESSION_VIDEOCHANNEL_SIZE"
 )

internal/server/handlers/sse.go

@@ -1,11 +1,13 @@
 package handlers
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/glimesh/broadcast-box/internal/environment"
 	"github.com/glimesh/broadcast-box/internal/server/helpers"
@@ -45,7 +47,9 @@ func sseHandler(responseWriter http.ResponseWriter, request *http.Request) {
 	for {
 		select {
 		case <-ctx.Done():
+			log.Println("API.SSE: Client disconnected")
 			return
+
 		case msg, ok := <-sseChannel:
 			if debugSseMessages {
 				log.Println("API.SSE Sending:", msg)
@@ -56,11 +60,30 @@ func sseHandler(responseWriter http.ResponseWriter, request *http.Request) {
 				return
 			}
 
-			if _, err := fmt.Fprintf(responseWriter, "%s\n", msg); err != nil {
-				log.Println("API.SSE Error:", err)
+			// Write with timeout
+			writeCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
+			done := make(chan error, 1)
+
+			go func() {
+				_, err := fmt.Fprintf(responseWriter, "%s\n", msg)
+				if err == nil {
+					flusher.Flush()
+				}
+				done <- err
+			}()
+
+			select {
+			case err := <-done:
+				cancel()
+				if err != nil {
+					log.Println("API.SSE Write error:", err)
+					return
+				}
+			case <-writeCtx.Done():
+				cancel()
+				log.Println("API.SSE Write timeout")
+				return
 			}
-
-			flusher.Flush()
 		}
 	}
 }

internal/webrtc/codecs/trackMultiCodec.go

@@ -7,6 +7,15 @@ import (
 	"github.com/pion/webrtc/v4"
 )
 
+type TrackPacket struct {
+	Layer        string
+	Packet       *rtp.Packet
+	TimeDiff     int64
+	SequenceDiff int
+	Codec        TrackCodeType
+	IsKeyframe   bool
+}
+
 type TrackMultiCodec struct {
 	id         string
 	rid        string

internal/webrtc/peerconnection/peerconnection_functions.go

@@ -68,18 +68,14 @@ func disconnected(isWhip bool, streamKey string, sessionId string) {
 	if isWhip {
 		log.Println("WhipSession.Disconnected: Removing tracks", sessionId)
 		whipSession.RemoveTracks()
-	} else {
-		log.Println("WhepSession.Disconnected: Removing session", sessionId)
-		//TODO: Find a way to use the SSE connection to be considdered the tether to an open connection
-		// whipSession.RemoveWhepSession(sessionId)
+
+		// Remove Whip session from manager if its empty
+		log.Println("WhipSession.RemoveWhipSession: No Whep session, closing down")
+		whipSession.ActiveContextCancel()
 	}
 
 	// Do not conclude stream if whep sessions are still listening, or the host is still active
 	if whipSession.HasWhepSessions() {
 		return
 	}
-
-	// Remove Whip session from manager if its empty
-	log.Println("WhipSession.RemoveWhipSession: No Whep session, closing down")
-	whipSession.ActiveContextCancel()
 }

internal/webrtc/peerconnection/peerconnection_whip_handlers.go

@@ -24,18 +24,10 @@ func RegisterWhipHandlers(whipSession *whip.WhipSession, peerConnection *webrtc.
 	whipSession.PeerConnection.OnConnectionStateChange(onConnectionStateChange(whipSession))
 }
 
-func onWhipICEConnectionStateChangeHandler(
-	streamKey string,
-	sessionId string,
-) func(webrtc.ICEConnectionState) {
-
+func onWhipICEConnectionStateChangeHandler(streamKey string, sessionId string) func(webrtc.ICEConnectionState) {
 	return func(state webrtc.ICEConnectionState) {
 		if state == webrtc.ICEConnectionStateFailed || state == webrtc.ICEConnectionStateClosed {
-
-			if strings.EqualFold("DEBUG_PEERCONNECTION_ENABLED", "true") {
-				log.Println("WhepSession: Disconnected", streamKey, sessionId)
-			}
-
+			log.Println("WhipSession.PeerConnection.OnICEConnectionStateChange", streamKey)
 			disconnected(true, streamKey, sessionId)
 		}
 	}
@@ -55,6 +47,12 @@ func onWhipTrackHandler(whipSession *whip.WhipSession, peerConnection *webrtc.Pe
 
 		// Fires when track has stopped
 		whipSession.OnTrackChangeChannel <- struct{}{}
+		err := peerConnection.Close()
+		if err != nil {
+			log.Println("WhipSession.OnTrackHandler.PeerConnection.Close.Error:", err)
+		}
+
+		log.Println("WhipSession.OnTrackHandler.TrackStopped", remoteTrack.RID())
 	}
 }
 
@@ -63,7 +61,9 @@ func onConnectionStateChange(whipSession *whip.WhipSession) func(webrtc.PeerConn
 		log.Println("WhipSession.PeerConnection.OnConnectionStateChange", state)
 
 		if state == webrtc.PeerConnectionStateDisconnected || state == webrtc.PeerConnectionStateClosed || state == webrtc.PeerConnectionStateFailed {
+			log.Println("WhipSession.PeerConnection.OnConnectionStateChange: Host removed")
 			whipSession.ActiveContextCancel()
+			whipSession.HasHost.Store(false)
 		}
 	}
 }