mtbf/endpoint.go

package main

import (
	"container/list"
	"errors"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

type Address struct {
	ip   string // TODO: switch to a static 16-byte array
	port int
	v6   bool
}

type EndpointState int

const (
	ES_Normal EndpointState = iota
	ES_Delayed
	ES_Deleted
)

// An Endpoint represents a remote target and stores its persistent data between multiple connections.
type Endpoint struct {
	addr Address // IP address of an endpoint

	loginPos, passwordPos SourcePos     // login/password cursors
	listElement           *list.Element // position in list

	state      EndpointState // which state an endpoint is in
	delayUntil time.Time     // when this endpoint can be used again

	// endpoint stats
	goodConn, badConn, protoErrors, readErrors int
	consecutiveGoodConn, consecutiveBadConn, consecutiveProtoErrors,
	consecutiveReadErrors int

	mutex sync.Mutex // sync primitive

	// unused, for now
	rtt             float32
	heuristicBanAPS int
	heuristicBanPPS int
	lastPacketAt    time.Time // when was the last packet sent?
	lastAttemptAt   time.Time // same, but for attempts
}

var endpoints *list.List        // Contains all active and ready endpoints
var delayedEndpoints *list.List // Contains endpoints that are active, but not ready

// A mutex for synchronizing Endpoint collections.
var globalEndpointMutex sync.Mutex

func (state EndpointState) String() string {
	switch state {
	case ES_Normal:
		return "normal"
	case ES_Delayed:
		return "delayed"
	case ES_Deleted:
		return "deleted"
	}

	return "unknown"
}

func (state EndpointState) GetList() *list.List {
	switch state {
	case ES_Normal:
		return endpoints
	case ES_Delayed:
		return delayedEndpoints
	}

	return nil
}

// String transforms an Endpoint to a string representation compatible with Dialer interface.
func (e *Endpoint) String() string {
	if e.addr.v6 {
		return "[" + e.addr.ip + "]:" + strconv.Itoa(e.addr.port)
	} else {
		return e.addr.ip + ":" + strconv.Itoa(e.addr.port)
	}
}

func (e *Endpoint) GetList() *list.List {
	return e.state.GetList()
}

// Delete deletes an endpoint from global storage.
// This method assumes that Endpoint's mutex was already taken.
func (e *Endpoint) Delete() {
	globalEndpointMutex.Lock()
	defer globalEndpointMutex.Unlock()

	list := e.GetList()
	if list != nil {
		log("ep", 3, "deleting endpoint \"%v\"", e)
		list.Remove(e.listElement)
		e.listElement = nil
	}

	e.delayUntil = time.Time{}
	e.state = ES_Deleted
}

// SetState changes an endpoint's state.
func (e *Endpoint) SetState(newState EndpointState) {
	if e.state == newState {
		log("ep", 5, "ignoring state change for an endpoint \"%v\": already in state \"%v\"", e, e.state)
		return
	}

	oldList := e.GetList()
	newList := newState.GetList()

	globalEndpointMutex.Lock()
	defer globalEndpointMutex.Unlock()

	if e.listElement != nil {
		oldList.Remove(e.listElement)
	}

	if newList == nil {
		e.listElement = nil
	} else {
		e.listElement = newList.PushBack(e)
	}
}

// Delay marks an Endpoint as "delayed" for a certain duration
// and migrates it to the delayed queue.
// This method assumes that Endpoint's mutex was already taken.
func (e *Endpoint) Delay(addTime time.Duration) {
	if e.state == ES_Normal {
		log("ep", 5, "delaying endpoint \"%v\" for %v", e, addTime)
		e.delayUntil = time.Now().Add(addTime)
		e.SetState(ES_Delayed)
	} else if e.state == ES_Delayed {
		// endpoints that are already delayed can have their delay time extended further

		tm := time.Now().Add(addTime)
		if e.delayUntil.Before(tm) {
			e.delayUntil = tm
		}
	}
}

// MigrateToNormal moves an Endpoint to a normal queue.
// Endpoint mutex is assumed to be taken.
func (e *Endpoint) MigrateToNormal() {
	endpointMutex.Lock()
	defer endpointMutex.Unlock()

	if e.normalList != nil {
		log("ep", 5, "cannot migrate endpoint \"%v\" to normal list: already in the list", e)
	} else {
		log("ep", 5, "migrating endpoint \"%v\" to normal list", e)
		e.normalList = endpoints.PushBack(e)
		if e.delayedList != nil {
			delayedEndpoints.Remove(e.delayedList)
			e.delayedList = nil
		}
	}
}

// SkipLogin gets the endpoint's current login,
// compares it with user-defined login and skips (advances) it if
// both logins are equal.
func (e *Endpoint) SkipLogin(login) {
	// attempt to fetch next login
	curLogin, empty := SrcLogin.FetchOne(&e.loginPos, false)
	if curLogin == login && !empty { // this login has not yet been exhausted?
		// reset password pos
		e.passwordPos.Reset()

		// fetch but ignore result
		SrcLogin.FetchOne(&e.loginPos, true)

		log("ep", 3, "advanced to next login for \"%v\"", e)
	}
}

// NoResponse is an event handler that gets called when
// an Endpoint does not respond to a connection request.
func (e *Endpoint) NoResponse() bool {
	e.mutex.Lock()
	defer e.mutex.Unlock()

	e.badConn++
	if e.consecutiveGoodConn == 0 {
		e.consecutiveBadConn++
	} else {
		e.consecutiveGoodConn = 0
		e.consecutiveBadConn = 1
	}

	// 1. always bail after X consecutive bad conns
	if e.consecutiveBadConn >= getParamInt("max-bad-conn") {
		log("ep", 3, "deleting \"%v\" due to max-bad-conn", e)
		e.Delete()
		return false
	}

	// 2. after a good conn, always allow at most X bad conns
	if e.goodConn > 0 && e.consecutiveBadConn <= getParamInt("max-bad-after-good-conn") {
		log("ep", 3, "keeping \"%v\" around due to max-bad-after-good-conn", e)
		e.Delay(getParamDurationMS("no-response-delay-ms"))
		return true
	}

	// 3. always allow at most X bad conns
	if e.consecutiveBadConn < getParamInt("min-bad-conn") {
		log("ep", 3, "keeping \"%v\" around due to min-bad-conn", e)
		e.Delay(getParamDurationMS("no-response-delay-ms"))
		return true
	}

	// 4. bad conn/good conn ratio must not be higher than X
	if e.goodConn > 0 && (float64(e.badConn)/float64(e.goodConn)) <= getParamFloat("conn-ratio") {
		log("ep", 3, "keeping \"%v\" around due to conn-ratio", e)
		e.Delay(getParamDurationMS("no-response-delay-ms"))
		return true
	}

	// otherwise, just delete it
	log("ep", 3, "deleting \"%v\" due to no applicable grace conditions", e)
	e.Delete()
	return false
}

// ProtocolError is an event handler that gets called when
// an Endpoint responds with wrong or missing data.
func (e *Endpoint) ProtocolError() bool {
	e.mutex.Lock()
	defer e.mutex.Unlock()

	e.protoErrors++
	e.consecutiveProtoErrors++

	// 1. always bail after X consecutive protocol errors
	if e.consecutiveProtoErrors >= getParamInt("max-proto-errors") {
		log("ep", 3, "deleting \"%v\" due to max-proto-errors", e)
		e.Delete()
		return false
	}

	// 2. always allow at most X consecutive protocol errors
	if e.consecutiveProtoErrors < getParamInt("min-proto-errors") {
		log("ep", 3, "keeping \"%v\" around due to min-proto-errors", e)
		e.Delay(getParamDurationMS("protocol-error-delay-ms"))
		return true
	}

	// 3. bad conn/good conn ratio must not be higher than X
	if e.goodConn > 0 && (float64(e.protoErrors)/float64(e.goodConn)) <= getParamFloat("proto-error-ratio") {
		log("ep", 3, "keeping \"%v\" around due to proto-error-ratio", e)
		e.Delay(getParamDurationMS("protocol-error-delay-ms"))
		return true
	}

	// otherwise, just delete it
	log("ep", 3, "deleting \"%v\" due to no applicable grace conditions", e)
	e.Delete()
	return false
}

// Bad is an event handler that gets called when
// an authentication attempt to an Endpoint fails.
func (e *Endpoint) Bad() {
	e.mutex.Lock()
	defer e.mutex.Unlock()
	e.consecutiveProtoErrors = 0

	// The endpoint may be in delayed queue, so push it back to the normal queue.
	e.MigrateToNormal()
}

// Good is an event handler that gets called when
// an authentication attempt to an Endpoint succeeds.
func (e *Endpoint) Good(login) {
	e.mutex.Lock()
	defer e.mutex.Unlock()
	e.consecutiveProtoErrors = 0

	if !getParamSwitch("keep-endpoint-on-good") {
		e.Delete()
	} else {
		e.MigrateToNormal()
		e.SkipLogin(login)
	}
}

// Connected is an event handler that gets called when
// a connection attempt to an Endpoint succeeds.
func (e *Endpoint) Connected() {
	e.mutex.Lock()
	defer e.mutex.Unlock()

	e.goodConn++
	if e.consecutiveBadConn == 0 {
		e.consecutiveGoodConn++
	} else {
		e.consecutiveBadConn = 0
		e.consecutiveGoodConn = 1
	}
}

// NoSuchLogin is an event handler that gets called when
// a service module determines that a login does not present
// on an Endpoint and therefore can be excluded from processing.
func (e *Endpoint) NoSuchLogin(login string) {
	e.mutex.Lock()
	defer e.mutex.Unlock()

	e.SkipLogin(login)
}

// EventWithParm tells an Endpoint that something important has happened,
// or a hint has been acquired.
// It is normally called from a Task handler.
// Returns False if an event resulted in a deletion of its Endpoint.
func (e *Endpoint) EventWithParm(event TaskEvent, parm any) bool {
	log("ep", 4, "endpoint event for \"%v\": %v", e, event)

	if event == TE_Generic {
		return true // do not process generic events
	}

	switch event {
	case TE_NoResponse:
		return e.NoResponse()

	case TE_ProtocolError:
		return e.ProtocolError()

	case TE_Good:
		e.Good(parm.(string))
		return false

	case TE_Bad:
		e.Bad()
	case TN_Connected:
		e.Connected()
	case TH_NoSuchLogin:
		e.NoSuchLogin(parm.(string))
	}

	return true // keep this endpoint
}

// Event is a parameterless version of EventWithParm.
func (e *Endpoint) Event(event TaskEvent) bool {
	return e.EventWithParm(event, 0)
}

// Exhausted gets called when an endpoint no longer has any valid logins and passwords,
// thus it may be deleted.
func (e *Endpoint) Exhausted() {
	e.mutex.Lock()
	defer e.mutex.Unlock()
	e.Delete()
}

// GetDelayedEndpoint retrieves an Endpoint from the delayed list.
// globalEndpointMutex must be already taken.
func GetDelayedEndpoint() (e *Endpoint, waitTime time.Duration) {
	currentTime := time.Now()

	if delayedEndpoints.Empty() {
		log("ep", 5, "delayed endpoint list is empty")
		return nil, 0
	}

	it := delayedEndpoints.IteratorAt(delayedEndpoints.Left())
	for {
		k, v := it.Key().(time.Time), it.Value().(*Endpoint)

		if v == nil {
			panic("delayed endpoint list contains an empty endpoint")
			return nil, 0
		}

		if k.After(currentTime) {
			log("ep", 5, "no delayed endpoints can be processed at this time")
			return nil, k.Sub(currentTime)
		}

		if k.Before(v.delayUntil) {
			log("ep", 5, "delayed endpoint was re-delayed: removing lingering definition")
			defer delayedEndpoints.Remove(k)
			it.Next()
			continue
		}

		if v.delayUntil.IsZero() {
			log("ep", 5, "delayed endpoint is already in normal queue: removing lingering definition")
			defer delayedEndpoints.Remove(k)
			it.Next()
			continue
		}

		defer delayedEndpoints.Remove(k)
		return v, 0
	}

	log("ep", 5, "delayed endpoint list was holding only lingering definitions and is now empty")
	return nil, 0
}

// FetchEndpoint retrieves an endpoint: first, a delayed list is queried,
// then, if nothing is found, a normal list is searched,
// and (TODO) if this list is empty or will soon be emptied,
// a new batch of endpoints gets created.
func FetchEndpoint() (e *Endpoint, waitTime time.Duration) {
	globalEndpointMutex.Lock()
	defer globalEndpointMutex.Unlock()

	log("ep", 4, "fetching an endpoint")

	e, waitTime = GetDelayedEndpoint()
	if e != nil {
		log("ep", 4, "fetched a delayed endpoint: \"%v\"", e)
		return e, 0
	}

	el := endpoints.Front()
	if el == nil {
		if waitTime == 0 {
			log("ep", 1, "out of endpoints")
			return nil, 0
		}

		log("ep", 4, "all endpoints are delayed, waiting for %v", waitTime)
		return nil, waitTime
	}

	endpoints.MoveToBack(el)
	e = el.Value.(*Endpoint)

	log("ep", 4, "fetched an endpoint: \"%v\"", e)
	return e, 0
}

// ---
// ---
// ---

// Safety feature, to avoid expanding subnets into a huge amount of IPs.
const maxNetmaskSize = 22 // expands into /10 for IPv4

// RegisterEndpoint builds an Endpoint and puts it to a global list of endpoints.
func RegisterEndpoint(ip string, ports []int, isIPv6 bool) int {
	for _, port := range ports {
		ep := Endpoint{addr: Address{ip: ip, port: port, v6: isIPv6}}
		ep.loginPos.Reset()
		ep.passwordPos.Reset()

		ep.listElement = endpoints.PushBack(&ep)
		log("ep", 3, "registered endpoint: %v", &ep)
	}

	return len(ports)
}

func incIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// parseCIDR registers multiple endpoints from a CIDR netmask.
func parseCIDR(ip string, ports []int, isIPv6 bool) int {
	na, nm, err := net.ParseCIDR(ip)
	if err != nil {
		log("ep", 0, "failed to parse CIDR notation for \"%v\": %v", ip, err.Error())
		return 0
	}

	mask, maskBits := nm.Mask.Size()
	if mask < maskBits-maxNetmaskSize {
		log("ep", 0, "ignoring out of safe bounds CIDR netmask for \"%v\": %v (max: %v, allowed: %v)", ip, mask, maskBits, maxNetmaskSize)
		return 0
	}

	curHost := 0
	maxHost := 1<<(maskBits-mask) - 1
	numParsed := 0
	strict := getParamSwitch("strict-subnets")

	log("ep", 2, "expanding CIDR: \"%v\" to %v hosts", ip, maxHost+1)

	for expIP := na.Mask(nm.Mask); nm.Contains(expIP); incIP(expIP) {
		if strict && (curHost == 0 || curHost == maxHost) && maskBits-mask >= 2 {
			log("ep", 1, "ignoring network/broadcast address due to strict-subnets: \"%v\"", expIP.String())
		} else {
			numParsed += RegisterEndpoint(expIP.String(), ports, isIPv6)
		}
		curHost++
	}

	return numParsed
}

// parseIPOrCIDR expands plain IP or CIDR to multiple endpoints.
func parseIPOrCIDR(ip string, ports []int, isIPv6 bool) int {
	// ip may be a domain name, a CIDR subnet or an IP address
	// CIDR subnets must be expanded to plain IPs

	if strings.LastIndex(ip, "/") >= 0 { // this is a CIDR subnet
		return parseCIDR(ip, ports, isIPv6)
	} else if strings.Count(ip, "/") > 1 { // invalid CIDR notation
		log("ep", 0, "invalid CIDR subnet format: \"%v\", ignoring", ip)
		return 0
	} else { // otherwise, just register
		return RegisterEndpoint(ip, ports, isIPv6)
	}
}

// extractIPAndPort extracts all endpoint components.
func extractIPAndPort(str string) (ip string, port int, err error) {
	var portString string
	ip, portString, err = net.SplitHostPort(str)
	if err != nil {
		return "", 0, err
	}

	port, err = strconv.Atoi(portString)
	if err != nil {
		return "", 0, err
	}

	if port <= 0 || port > 65535 {
		return "", 0, errors.New("invalid port: " + strconv.Itoa(port))
	}

	return ip, port, nil
}

// ParseEndpoints takes a string slice of IPs/CIDR subnets and converts it to a list of endpoints.
func ParseEndpoints(source []string) {
	log("ep", 1, "parsing endpoints")

	totalIPv6Skipped := 0
	numParsed := 0

	for _, str := range source {
		if !strings.Contains(str, ":") {
			// no ":": this is an ipv4/dn without port,
			// parse it with all known ports

			numParsed += parseIPOrCIDR(str, getParamIntSlice("port"), false)
		} else {
			// either ipv4/dn with port, or ipv6 with/without port

			isIPv6 := strings.Count(str, ":") > 1
			if isIPv6 && getParamSwitch("no-ipv6") {
				totalIPv6Skipped++
				continue
			}

			if !strings.Contains(str, "]:") && strings.Contains(str, "::") {
				// ipv6 without port
				numParsed += parseIPOrCIDR(str, getParamIntSlice("port"), true)
				continue
			}

			ip, port, err := extractIPAndPort(str)
			if err != nil {
				log("ep", 0, "failed to extract ip/port for \"%v\": %v, ignoring endpoint", str, err.Error())
				continue
			}

			ports := []int{port}
			// append all default ports
			if getParamSwitch("append-default-ports") {
				for _, port2 := range getParamIntSlice("port") {
					if port != port2 {
						ports = append(ports, port2)
					}
				}
			}

			numParsed += parseIPOrCIDR(ip, ports, isIPv6)
		}
	}

	logIf(totalIPv6Skipped > 0, "ep", 0, "skipped %v IPv6 targets due to no-ipv6 flag", totalIPv6Skipped)
	log("ep", 1, "finished parsing endpoints: parsed %v out of total %v", numParsed, endpoints.Len())
}

func init() {
	endpoints = list.New()
	delayedEndpoints = list.New()

	registerParam("port", []int{8291}, "one or more default ports")
	registerParam("max-aps", 5, "maximum number of attempts per second for an endpoint")
	registerSwitch("no-ipv6", "skip IPv6 entries")
	registerSwitch("append-default-ports", "always append default ports even for targets in host:port format")
	registerSwitch("strict-subnets", "strict subnet behaviour: ignore network and broadcast addresses in /30 and bigger subnets")

	registerSwitch("keep-endpoint-on-good", "keep processing endpoint if a login/password was found")

	registerParam("conn-ratio", 0.15, "keep a failed endpoint if its bad/good connection ratio is lower than this value")
	registerParam("max-bad-after-good-conn", 5, "how many consecutive bad connections to allow after a good connection")
	registerParam("max-bad-conn", 20, "always remove endpoint after this many consecutive bad connections")
	registerParam("min-bad-conn", 2, "do not consider removing an endpoint if it does not have this many consecutive bad connections")

	registerParam("proto-error-ratio", 0.25, "keep endpoints with a protocol error if their protocol error ratio is lower than this value")
	registerParam("max-proto-errors", 20, "always remove endpoint after this many consecutive protocol errors")
	registerParam("min-proto-errors", 4, "do not consider removing an endpoint if it does not have this many consecutive protocol errors")

	registerParam("read-error-ratio", 0.25, "keep endpoints with a read error if their read error ratio is lower than this value")
	registerParam("max-read-errors", 20, "always remove endpoint after this many consecutive read errors")
	registerParam("min-read-errors", 3, "do not consider removing an endpoint if it does not have this many consecutive read errors")

	registerParam("no-response-delay-ms", 2000, "wait for this number of ms if an endpoint does not respond")
	registerParam("read-error-delay-ms", 5000, "wait for this number of ms if an endpoint returns a read error")
	registerParam("protocol-error-delay-ms", 5000, "wait for this number of ms if an endpoint returns a protocol error")
}