seekia/internal/moderation/verifiedStickyStatus/verifiedStickyStatus.go

// verifiedStickyStatus provides functions to determine the sticky consensus viewable status for profiles/identities/messages
// Sticky consensus statuses are a kind of consensus status that requires a verdict to be present for a minimum defined period of time

package verifiedStickyStatus

// Sticky consensus is needed to defend against malicious moderators
// Here is an example:
// A malicious moderator bans all other moderators and all content on the network. All profiles on the network are now banned.
// Other moderators need to ban this moderator to undo the damage.
// Without sticky consensus, all the hosts would treat all network profiles as being banned, and would stop seeding these profiles to users
// This single malicious moderator could cripple the network for as long as it would take to ban that moderator
// Banning this moderator could take hours, and is more difficult the more highly ranked they are.

// Sticky consensus attempts to solve this problem
// With sticky consensus, as long as a profile has been approved for a certain period of time, its sticky consensus will be stuck
// For the sticky consensus to be switched to unviewable, the profile's verdict would have to be Ban for a certain period of time
// Hosts will serve profiles to users based on each profile's sticky viewable consensus status, not its realtime consensus verdict

// Hosts must be online for long enough to keep track of, or establish, the sticky consensus for content within their ranges
// Each sticky status can only be considered established if the user's client has been downloading the content's reviews for long enough
// This establishing time is needed for several reasons:
//	1. When adding a new range, the host needs time to initially download the reviews for content within the range.
//	2. Hosts may initially get an inaccurate view of the sticky consensus due to malicious hosts or hosts that are not caught up with the rest of the network
//	3. The status may have only recently been flipped by many malicious moderators, and will be flipped back to the "true" consensus after those malicious moderators are banned. The host would only see a small portion of the true verdict history. 
// 	-For example, During the last 5 minutes, a profile was viewable 100% of the time
//		-But within the last 50 minutes, it has been viewable for only 10% of the time
//
// Hosts will only share a profile/message/identity's sticky consensus to requesting peers once the consensus is established

// Each ReviewedType has two sticky status states: Viewable and Unviewable
// Below describes what defines a viewable/unviewable status for each reviewedType:

// Identity:
//	-Viewable: Not Banned
//	-Unviewable: Banned
// Profile
//	-Viewable:
//		-Mate: Approved
//		-Host/Moderator: Approved/Undecided
//	-Unviewable
//		-Mate: Banned/Undecided
//		-Host/Moderator: Banned
// Message:
//	-Viewable: Approved/Undecided
//	-Unviewable: Banned

// The calculation works as follows:
// The host keeps track of the verdict history for all downloaded and reviewed identities/profiles/messages at a regular interval
// To determine the current sticky status for a reviewedHash, we take all statuses between currentTime and currentTime - HistoricalExpirationTime
// We determine the percentage of all statuses within this set that are viewable/unviewable
// We check to see if the percentage is greater than the minimum viewable percentage for the reviewedType
// If the percentage of viewable statuses is greater than this percentage, the status is viewable. Otherwise, the status is unviewable.
// This must be an uninterrupted period of verdicts. If there is a gap in verdicts, the percentage is reset.

// There are 9 variables within the moderation parameters that define the sticky status calculation
//	-StatusEstablishingTime for each reviewedType
// 		-This is the amount of time a host must wait before they start to rely upon and share their sticky statuses to the network
//		-The host must be downloading the reviewedType's reviews for this long
//	-VerdictExpirationTime for each reviewedType
//		-This is the amount of time that historical statuses will be kept and have an impact on the sticky consensus
//	-MinimumViewablePercentage for each reviewedType
//		-This is the percentage of time that the content must be viewable for the sticky status to be viewable

// The trusted profileIsApproved sticky status is indicative of the identityIsBanned status
// The verified profileIsApproved sticky status is not.
// If a profile's author is banned but the profile is approved, the profile's trusted sticky consensus will be Unviewable, but the profile's verified sticky consensus will be Viewable

import "seekia/internal/contentMetadata"
import "seekia/internal/encoding"
import "seekia/internal/helpers"
import "seekia/internal/moderation/verdictHistory"
import "seekia/internal/network/appNetworkType/getAppNetworkType"
import "seekia/internal/network/backgroundDownloads"
import "seekia/internal/profiles/readProfiles"

import "slices"
import "time"
import "sync"
import "errors"

// These maps should only store the sticky statuses for identities/content on the current app network type

// Map structure: Identity Hash -> Current sticky consensus viewable status
var identityStickyStatusesMap map[[16]byte]bool = make(map[[16]byte]bool)
var identityStickyStatusesMapMutex sync.RWMutex

// Map structure: Profile Hash -> Current sticky consensus viewable status
var profileStickyStatusesMap map[[28]byte]bool = make(map[[28]byte]bool)
var profileStickyStatusesMapMutex sync.RWMutex

// Map structure: Message Hash -> Current sticky consensus viewable status
var messageStickyStatusesMap map[[26]byte]bool = make(map[[26]byte]bool)
var messageStickyStatusesMapMutex sync.RWMutex


// This function must be called whenever the app network type is switched
func DeleteVerifiedStickyStatuses(){

	identityStickyStatusesMapMutex.Lock()
	clear(identityStickyStatusesMap)
	identityStickyStatusesMapMutex.Unlock()

	profileStickyStatusesMapMutex.Lock()
	clear(profileStickyStatusesMap)
	profileStickyStatusesMapMutex.Unlock()

	messageStickyStatusesMapMutex.Lock()
	clear(messageStickyStatusesMap)
	messageStickyStatusesMapMutex.Unlock()
}


//Outputs:
//	-bool: Downloading required reviews and moderator profiles
//	-bool: Necessary parameters exist
//	-bool: Sticky consensus status established
//	-bool: Sticky Identity is viewable consensus status (true == identity is viewable, false == Identity is not viewable)
//	-error
func GetVerifiedIdentityIsViewableStickyStatus(identityHash [16]byte, networkType byte)(bool, bool, bool, bool, error){

	isValid := helpers.VerifyNetworkType(networkType)
	if (isValid == false){
		networkTypeString := helpers.ConvertByteToString(networkType)
		return false, false, false, false, errors.New("GetVerifiedIdentityIsViewableStickyStatus called with invalid networkType: " + networkTypeString)
	}

	clientIsDownloadingRequiredReviews, err := 	backgroundDownloads.CheckIfAppCanDetermineIdentityVerdicts(identityHash)
	if (err != nil) { return false, false, false, false, err }
	if (clientIsDownloadingRequiredReviews == false){
		return false, false, false, false, nil
	}

	appNetworkType, err := getAppNetworkType.GetAppNetworkType()
	if (err != nil) { return false, false, false, false, err }
	if (appNetworkType != networkType){
		// We are not downloading the reviews for this network type.
		return false, false, false, false, nil
	}

	//TODO: Fix below
	parametersExist := true
	if (parametersExist == false){

		return true, false, false, false, nil
	}

	identityStickyStatusesMapMutex.RLock()
	currentViewableStatus, exists := identityStickyStatusesMap[identityHash]
	identityStickyStatusesMapMutex.RUnlock()
	if (exists == false){
		return true, true, false, false, nil
	}

	return true, true, true, currentViewableStatus, nil	
}

//Outputs:
//	-bool: Profile is disabled
//	-bool: Profile metadata is known
//	-byte: Profile network type
//	-[16]byte: Profile author identity hash
//	-bool: Client is downloading required reviews and moderator profiles
//	-bool: Necessary parameters exist
//	-bool: Sticky consensus status established
//	-bool: Sticky consensus viewable status (true == Viewable, false == Unviewable)
//	-error
func GetVerifiedProfileIsViewableStickyStatus(profileHash [28]byte)(bool, bool, byte, [16]byte, bool, bool, bool, bool, error){

	_, profileIsDisabled, err := readProfiles.ReadProfileHashMetadata(profileHash)
	if (err != nil) {
		profileHashHex := encoding.EncodeBytesToHexString(profileHash[:])
		return false, false, 0, [16]byte{}, false, false, false, false, errors.New("GetVerifiedProfileIsViewableStickyStatus called with invalid profileHash: " + profileHashHex)
	}

	metadataExists, _, profileNetworkType, profileAuthor, _, profileIsDisabledB, _, _, err := contentMetadata.GetProfileMetadata(profileHash)
	if (err != nil) { return false, false, 0, [16]byte{}, false, false, false, false, err }
	if (metadataExists == false){
		return profileIsDisabled, false, 0, [16]byte{}, false, false, false, false, nil
	}
	if (profileIsDisabled != profileIsDisabledB){
		return false, false, 0, [16]byte{}, false, false, false, false, errors.New("GetProfileMetadata returning different profileIsDisabled status.")
	}
	if (profileIsDisabled == true){
		return true, true, profileNetworkType, profileAuthor, false, false, false, false, nil
	}

	clientIsDownloadingRequiredReviews, err := backgroundDownloads.CheckIfAppCanDetermineIdentityVerdicts(profileAuthor)
	if (err != nil) { return false, false, 0, [16]byte{}, false, false, false, false, err }
	if (clientIsDownloadingRequiredReviews == false){

		return false, true, profileNetworkType, profileAuthor, false, false, false, false, nil
	}

	appNetworkType, err := getAppNetworkType.GetAppNetworkType()
	if (err != nil) { return false, false, 0, [16]byte{}, false, false, false, false, err }
	if (appNetworkType != profileNetworkType){
		// We are not downloading the reviews for this profile.
		return false, true, profileNetworkType, profileAuthor, false, false, false, false, nil
	}

	//TODO: Fix below
	parametersExist := true
	if (parametersExist == false){
		return false, true, profileNetworkType, profileAuthor, true, false, false, false, nil
	}

	profileStickyStatusesMapMutex.RLock()
	currentViewableStatus, exists := profileStickyStatusesMap[profileHash]
	profileStickyStatusesMapMutex.RUnlock()
	if (exists == false){
		return false, true, profileNetworkType, profileAuthor, true, true, false, false, nil
	}

	return false, true, profileNetworkType, profileAuthor, true, true, true, currentViewableStatus, nil
}


//Outputs:
//	-bool: Message metadata is known (inbox/cipher key hash)
//	-byte: Message network type
//	-[10]byte: Message inbox
//	-[25]byte: Message cipher key hash
//	-bool: Client is downloading required reviews and moderator profiles
//	-bool: Necessary Parameters exist
//	-bool: Message sticky consensus status established
//	-bool: Message sticky consensus viewable status (true == Viewable, false == Unviewable)
//	-error
func GetVerifiedMessageIsViewableStickyStatus(messageHash [26]byte)(bool, byte, [10]byte, [25]byte, bool, bool, bool, bool, error){

	metadataExists, _, messageNetworkType, _, messageInbox, messageCipherKeyHash, err := contentMetadata.GetMessageMetadata(messageHash)
	if (err != nil) { return false, 0, [10]byte{}, [25]byte{}, false, false, false, false, err }
	if (metadataExists == false){
		return false, 0, [10]byte{}, [25]byte{}, false, false, false, false, nil
	}

	clientIsDownloadingRequiredReviews, err := backgroundDownloads.CheckIfAppCanDetermineMessageVerdict(messageNetworkType, messageInbox, true, messageHash)
	if (err != nil) { return false, 0, [10]byte{}, [25]byte{}, false, false, false, false, err }
	if (clientIsDownloadingRequiredReviews == false){

		return true, messageNetworkType, messageInbox, messageCipherKeyHash, false, false, false, false, nil
	}

	//TODO: Fix below
	parametersExist := true
	if (parametersExist == false){
		return true, messageNetworkType, messageInbox, messageCipherKeyHash, true, false, false, false, nil
	}

	messageStickyStatusesMapMutex.RLock()
	currentViewableStatus, exists := messageStickyStatusesMap[messageHash]
	messageStickyStatusesMapMutex.RUnlock()
	if (exists == false){
		return true, messageNetworkType, messageInbox, messageCipherKeyHash, true, true, false, false, nil
	}

	return true, messageNetworkType, messageInbox, messageCipherKeyHash, true, true, true, currentViewableStatus, nil
}

func UpdateIdentityStickyStatusesMap()error{

	allReviewedIdentityHashesList, err := verdictHistory.GetAllReviewedIdentityHashesList()
	if (err != nil) { return err }

	//TODO: Fix below to retrieve from parameters
	statusEstablishingTime := int64(100)
	verdictExpirationTime := int64(1000)
	minimumViewablePercentage := 60

	for _, identityHash := range allReviewedIdentityHashesList{

		identityVerdictHistoryMap, err := verdictHistory.GetIdentityVerdictHistoryMap(identityHash)
		if (err != nil) { return err }

		statusIsEstablished, isViewableStatus, err := getStickyViewableStatusFromVerdictHistoryMap(identityVerdictHistoryMap, false, statusEstablishingTime, verdictExpirationTime, minimumViewablePercentage)
		if (err != nil) { return err }

		identityStickyStatusesMapMutex.Lock()

		if (statusIsEstablished == false){
			delete(identityStickyStatusesMap, identityHash)
		} else {
			identityStickyStatusesMap[identityHash] = isViewableStatus
		}

		identityStickyStatusesMapMutex.Unlock()
	}

	return nil
}


func UpdateProfileStickyStatusesMap()error{

	allReviewedProfileHashesList, err := verdictHistory.GetAllReviewedProfileHashesList()
	if (err != nil) { return err }

	//TODO: Fix below to retrieve from parameters
	statusEstablishingTime := int64(100)
	verdictExpirationTime := int64(1000)
	minimumViewablePercentage := 60

	for _, profileHash := range allReviewedProfileHashesList{

		getIsMateProfileBool := func()(bool, error){

			profileType, _, err := readProfiles.ReadProfileHashMetadata(profileHash)
			if (err != nil) { return false, err }
			if (profileType == "Mate"){
				return true, nil
			}
			return false, nil
		}

		isMateProfile, err := getIsMateProfileBool()
		if (err != nil) { return err }

		profileVerdictHistoryMap, err := verdictHistory.GetProfileVerdictHistoryMap(profileHash)
		if (err != nil) { return err }

		statusIsEstablished, isViewableStatus, err := getStickyViewableStatusFromVerdictHistoryMap(profileVerdictHistoryMap, isMateProfile, statusEstablishingTime, verdictExpirationTime, minimumViewablePercentage)
		if (err != nil) { return err }

		profileStickyStatusesMapMutex.Lock()

		if (statusIsEstablished == false){
			delete(profileStickyStatusesMap, profileHash)
		} else {
			profileStickyStatusesMap[profileHash] = isViewableStatus
		}

		profileStickyStatusesMapMutex.Unlock()
	}
	
	return nil
}


func UpdateMessageStickyStatusesMap()error{

	allReviewedMessageHashesList, err := verdictHistory.GetAllReviewedMessageHashesList()
	if (err != nil) { return err }

	//TODO: Fix below to retrieve from parameters
	statusEstablishingTime := int64(100)
	verdictExpirationTime := int64(1000)
	minimumViewablePercentage := 60

	for _, messageHash := range allReviewedMessageHashesList{

		messageVerdictHistoryMap, err := verdictHistory.GetMessageVerdictHistoryMap(messageHash)
		if (err != nil) { return err }

		statusIsEstablished, isViewableStatus, err := getStickyViewableStatusFromVerdictHistoryMap(messageVerdictHistoryMap, false, statusEstablishingTime, verdictExpirationTime, minimumViewablePercentage)
		if (err != nil) { return err }

		messageStickyStatusesMapMutex.Lock()

		if (statusIsEstablished == false){
			delete(messageStickyStatusesMap, messageHash)
		} else {
			messageStickyStatusesMap[messageHash] = isViewableStatus
		}

		messageStickyStatusesMapMutex.Unlock()
	}
	
	return nil
}


func PruneOldConsensusStatuses()error{

	// TODO: Prune statuses for content that is expired and more...

	return nil
}

//Inputs:
//	-map[int64]string: Unix Time -> Verdict at unixTime
//	-bool: ReviewedHash is a mate profile
//	-int64: Time required for a status to be established. Verdicts must exist on a regular interval for this duration
//	-int64: The length of time that a verdict will have an impact on the sticky viewable status
//	-int: The percentage of verdicts that must be viewable for the status to be considered viewable
//		-Example: 60 = 60% of statuses must be viewable for the sticky status to be viewable
//Outputs:
//	-bool: Status is established
//	-bool: Profile/Message/Identity is viewable
//	-error
func getStickyViewableStatusFromVerdictHistoryMap(verdictHistoryMap map[int64]string, isMateProfile bool, establishingTime int64, verdictExpirationTime int64, minimumViewablePercentage int)(bool, bool, error){

	currentTime := time.Now().Unix()
	oldestEligibleVerdictTime := currentTime - verdictExpirationTime

	verdictTimesList := make([]int64, 0)

	for verdictTime, _ := range verdictHistoryMap{

		if (verdictTime < oldestEligibleVerdictTime){
			// We do not count verdicts that are this old in our stickyConsensus calculation
			continue
		}

		verdictTimesList = append(verdictTimesList, verdictTime)
	}
	
	if (len(verdictTimesList) <= 2){
		// Not enough verdict to calculate sticky consensus status
		return false, false, nil
	}

	// We sort verdicts in ascending order, oldest to newest

	slices.Sort(verdictTimesList)

	// Now we iterate through verdicts
	// We record the number of verdicts and the number of verdicts that are viewable
	// If we encounter a long enough break between verdicts, we reset our counts
	// This is done because we need an uninterrupted period of verdicts to make our calculation

	numberOfEligibleVerdicts := 0
	numberOfViewableVerdicts := 0

	// We keep track of the oldest verdict time to determine the beginning of the continuous period of verdicts
	// If the period is not old enough, the consensus status is not established
	oldestVerdictTime := verdictTimesList[0]

	// We keep track of previous verdict time to check the length of gaps between verdict recordings
	previousVerdictTime := int64(0)

	for index, verdictTime := range verdictTimesList{

		if (index != 0){

			timeElapsedSinceLastVerdict := verdictTime - previousVerdictTime

			if (timeElapsedSinceLastVerdict > (verdictHistory.VerdictIntervalDuration * 2)){
				// There was too long of a gap between verdict recordings
				// We reset our counts
				numberOfEligibleVerdicts = 0
				numberOfViewableVerdicts = 0
				oldestVerdictTime = verdictTime
			}
		}
		previousVerdictTime = verdictTime

		numberOfEligibleVerdicts += 1

		verdict, exists := verdictHistoryMap[verdictTime]
		if (exists == false){
			return false, false, errors.New("verdictHistoryMap missing verdictTime")
		}

		checkIfVerdictIsViewable := func()bool{

			if (verdict == "Banned"){
				return false
			}

			if (verdict == "Approved"){
				return true
			}

			// verdict == "Undecided"
			// reviewedHash type is not identity

			if (isMateProfile == true){
				// Mate profiles must be approved to be viewable. "Undecided" is not enough.
				return false		
			}

			// ReviewedHash is non-mate profile/message. These are only not viewable if they are banned
			return true
		}

		viewableStatus := checkIfVerdictIsViewable()

		if (viewableStatus == true){
			numberOfViewableVerdicts += 1
		}
	}

	if (numberOfEligibleVerdicts == 0){
		// No eligible verdicts exist
		return false, false, nil
	}

	// This represents the amount of time we have been recording verdicts without a gap that is too long
	trackedDuration := currentTime - oldestVerdictTime

	if (trackedDuration < establishingTime){
		// We have not been tracking verdicts for long enough to determine the sticky viewable status
		// The sticky consensus will be established once the client has been enabled for long enough
		return false, false, nil
	}

	// Sticky viewable status is established.
	// We see if status is viewable or not.

	percentageOfViewableVerdicts := (float64(numberOfViewableVerdicts)/float64(numberOfEligibleVerdicts)) * 100

	if (percentageOfViewableVerdicts < float64(minimumViewablePercentage)){
		return true, false, nil
	}
	return true, true, nil
}