461 lines
14 KiB
Go
461 lines
14 KiB
Go
|
|
// verdictHistory provides functions to keep track of and retrieve the verdict history for messages, profiles, and identities
|
|
|
|
package verdictHistory
|
|
|
|
// Verdict histories are used for 2 things:
|
|
|
|
// 1. Sticky consensus statuses are calculated using the verdict history of a reviewedHash
|
|
|
|
// 2. Hosts use the verdict history of a reviewedHash to know when to delete banned content
|
|
// Once a profile/identity/message has been banned for a long enough time period, the host's client will delete the content
|
|
|
|
//TODO: Add job to prune verdicts we no longer need
|
|
//TODO: Store the verdict history as messagePack files, so if a user briefly closes client, history is maintained upon restart
|
|
|
|
import "seekia/internal/badgerDatabase"
|
|
import "seekia/internal/helpers"
|
|
import "seekia/internal/moderation/verifiedVerdict"
|
|
import "seekia/internal/profiles/profileStorage"
|
|
|
|
|
|
import "errors"
|
|
import "time"
|
|
import "sync"
|
|
|
|
// Map Structure: Identity Hash -> Last time the verdict was recorded for this identityHash in the identityVerdictHistoryMap
|
|
var identityVerdictLastRecordedMap map[[16]byte]int64 = make(map[[16]byte]int64)
|
|
var identityVerdictLastRecordedMapMutex sync.RWMutex
|
|
|
|
// Map Structure: Profile Hash -> Last time the verdict was recorded for this profileHash in the profileVerdictHistoryMap
|
|
var profileVerdictLastRecordedMap map[[28]byte]int64 = make(map[[28]byte]int64)
|
|
var profileVerdictLastRecordedMapMutex sync.RWMutex
|
|
|
|
// Map Structure: Message Hash -> Last time the verdict was recorded for this messageHash in the messageVerdictHistoryMap
|
|
var messageVerdictLastRecordedMap map[[26]byte]int64 = make(map[[26]byte]int64)
|
|
var messageVerdictLastRecordedMapMutex sync.RWMutex
|
|
|
|
type verdictObject struct{
|
|
|
|
// Consensus verdict ("Approved"/"Banned"/"Undecided"/"Not Banned")
|
|
Verdict string
|
|
|
|
VerdictTime int64
|
|
}
|
|
|
|
// Map Structure: Identity hash -> List of verdict objects
|
|
var identityVerdictHistoryMap map[[16]byte][]verdictObject = make(map[[16]byte][]verdictObject)
|
|
var identityVerdictHistoryMapMutex sync.RWMutex
|
|
|
|
// Map Structure: Profile hash -> List of verdict objects
|
|
var profileVerdictHistoryMap map[[28]byte][]verdictObject = make(map[[28]byte][]verdictObject)
|
|
var profileVerdictHistoryMapMutex sync.RWMutex
|
|
|
|
// Map Structure: Message hash -> List of verdict objects
|
|
var messageVerdictHistoryMap map[[26]byte][]verdictObject = make(map[[26]byte][]verdictObject)
|
|
var messageVerdictHistoryMapMutex sync.RWMutex
|
|
|
|
|
|
// This is the minimum duration in seconds between recording the consensus verdicts for each reviewedHash
|
|
const VerdictIntervalDuration int64 = 180
|
|
|
|
|
|
// This function should be called whenever we switch network types
|
|
func DeleteVerdictHistory(){
|
|
|
|
identityVerdictLastRecordedMapMutex.Lock()
|
|
clear(identityVerdictLastRecordedMap)
|
|
identityVerdictLastRecordedMapMutex.Unlock()
|
|
|
|
profileVerdictLastRecordedMapMutex.Lock()
|
|
clear(profileVerdictLastRecordedMap)
|
|
profileVerdictLastRecordedMapMutex.Unlock()
|
|
|
|
messageVerdictLastRecordedMapMutex.Lock()
|
|
clear(messageVerdictLastRecordedMap)
|
|
messageVerdictLastRecordedMapMutex.Unlock()
|
|
|
|
identityVerdictHistoryMapMutex.Lock()
|
|
clear(identityVerdictHistoryMap)
|
|
identityVerdictHistoryMapMutex.Unlock()
|
|
|
|
profileVerdictHistoryMapMutex.Lock()
|
|
clear(profileVerdictHistoryMap)
|
|
profileVerdictHistoryMapMutex.Unlock()
|
|
|
|
messageVerdictHistoryMapMutex.Lock()
|
|
clear(messageVerdictHistoryMap)
|
|
messageVerdictHistoryMapMutex.Unlock()
|
|
}
|
|
|
|
|
|
//Outputs:
|
|
// -[][16]byte: Reviewed identity hashes list
|
|
// -error
|
|
func GetAllReviewedIdentityHashesList()([][16]byte, error){
|
|
|
|
identityVerdictLastRecordedMapMutex.RLock()
|
|
|
|
allReviewedIdentityHashesList := helpers.GetListOfMapKeys(identityVerdictLastRecordedMap)
|
|
|
|
identityVerdictLastRecordedMapMutex.RUnlock()
|
|
|
|
return allReviewedIdentityHashesList, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -[][28]byte: Reviewed profile hashes list
|
|
// -error
|
|
func GetAllReviewedProfileHashesList()([][28]byte, error){
|
|
|
|
profileVerdictLastRecordedMapMutex.RLock()
|
|
|
|
allReviewedProfileHashesList := helpers.GetListOfMapKeys(profileVerdictLastRecordedMap)
|
|
|
|
profileVerdictLastRecordedMapMutex.RUnlock()
|
|
|
|
return allReviewedProfileHashesList, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -[][26]byte: Reviewed message hashes list
|
|
// -error
|
|
func GetAllReviewedMessageHashesList()([][26]byte, error){
|
|
|
|
messageVerdictLastRecordedMapMutex.RLock()
|
|
|
|
allReviewedMessageHashesList := helpers.GetListOfMapKeys(messageVerdictLastRecordedMap)
|
|
|
|
messageVerdictLastRecordedMapMutex.RUnlock()
|
|
|
|
return allReviewedMessageHashesList, nil
|
|
}
|
|
|
|
|
|
//Outputs:
|
|
// -map[int64]string: Verdict history map
|
|
// -error
|
|
func GetIdentityVerdictHistoryMap(identityHash [16]byte)(map[int64]string, error){
|
|
|
|
identityVerdictHistoryMapMutex.RLock()
|
|
defer identityVerdictHistoryMapMutex.RUnlock()
|
|
|
|
verdictObjectsList, exists := identityVerdictHistoryMap[identityHash]
|
|
if (exists == false){
|
|
|
|
emptyMap := make(map[int64]string)
|
|
return emptyMap, nil
|
|
}
|
|
|
|
verdictHistoryMap, err := getReviewedHashVerdictHistoryMap(verdictObjectsList)
|
|
if (err != nil) { return nil, err }
|
|
|
|
return verdictHistoryMap, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -map[int64]string: Verdict history map
|
|
// -error
|
|
func GetProfileVerdictHistoryMap(profileHash [28]byte)(map[int64]string, error){
|
|
|
|
profileVerdictHistoryMapMutex.RLock()
|
|
defer profileVerdictHistoryMapMutex.RUnlock()
|
|
|
|
verdictObjectsList, exists := profileVerdictHistoryMap[profileHash]
|
|
if (exists == false){
|
|
|
|
emptyMap := make(map[int64]string)
|
|
return emptyMap, nil
|
|
}
|
|
|
|
verdictHistoryMap, err := getReviewedHashVerdictHistoryMap(verdictObjectsList)
|
|
if (err != nil) { return nil, err }
|
|
|
|
return verdictHistoryMap, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -map[int64]string: Verdict history map
|
|
// -error
|
|
func GetMessageVerdictHistoryMap(messageHash [26]byte)(map[int64]string, error){
|
|
|
|
messageVerdictHistoryMapMutex.RLock()
|
|
defer messageVerdictHistoryMapMutex.RUnlock()
|
|
|
|
verdictObjectsList, exists := messageVerdictHistoryMap[messageHash]
|
|
if (exists == false){
|
|
|
|
emptyMap := make(map[int64]string)
|
|
return emptyMap, nil
|
|
}
|
|
|
|
verdictHistoryMap, err := getReviewedHashVerdictHistoryMap(verdictObjectsList)
|
|
if (err != nil) { return nil, err }
|
|
|
|
return verdictHistoryMap, nil
|
|
}
|
|
|
|
func getReviewedHashVerdictHistoryMap(verdictObjectsList []verdictObject)(map[int64]string, error){
|
|
|
|
// Map Structure: Unix Time -> Verdict of reviewedHash at unix time
|
|
reviewedHashVerdictHistoryMap := make(map[int64]string)
|
|
|
|
for _, verdictObject := range verdictObjectsList{
|
|
|
|
verdict := verdictObject.Verdict
|
|
verdictTime := verdictObject.VerdictTime
|
|
|
|
reviewedHashVerdictHistoryMap[verdictTime] = verdict
|
|
}
|
|
|
|
return reviewedHashVerdictHistoryMap, nil
|
|
}
|
|
|
|
// This is run automatically by backgroundJobs on an internal
|
|
func RecordIdentityVerdictsToHistoryMap(networkType byte)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("RecordIdentityVerdictsToHistoryMap called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
allDownloadedProfileIdentityHashes, err := profileStorage.GetAllStoredProfileIdentityHashes()
|
|
if (err != nil) { return err }
|
|
|
|
allReviewedIdentityHashes, err := badgerDatabase.GetAllReviewedIdentityHashes()
|
|
if (err != nil) { return err }
|
|
|
|
allUserIdentityHashesList := helpers.CombineTwoListsAndAvoidDuplicates(allDownloadedProfileIdentityHashes, allReviewedIdentityHashes)
|
|
|
|
for _, identityHash := range allUserIdentityHashesList{
|
|
|
|
currentTime := time.Now().Unix()
|
|
|
|
identityVerdictLastRecordedMapMutex.RLock()
|
|
verdictLastRecordedTime, exists := identityVerdictLastRecordedMap[identityHash]
|
|
identityVerdictLastRecordedMapMutex.RUnlock()
|
|
if (exists == true){
|
|
|
|
timeElapsed := currentTime - verdictLastRecordedTime
|
|
if (timeElapsed < VerdictIntervalDuration){
|
|
// We have recorded this consensus verdict recently enough.
|
|
// We go to the next identity
|
|
continue
|
|
}
|
|
}
|
|
|
|
downloadingRequiredReviews, parametersExist, identityIsBanned, _, _, _, err := verifiedVerdict.GetVerifiedIdentityVerdict(identityHash, networkType)
|
|
if (err != nil) { return err }
|
|
if (downloadingRequiredReviews == false || parametersExist == false){
|
|
// We do not know consensus verdict
|
|
// We go to the next identity
|
|
continue
|
|
}
|
|
|
|
getVerdictString := func()string{
|
|
if (identityIsBanned == true){
|
|
return "Banned"
|
|
}
|
|
return "Not Banned"
|
|
}
|
|
|
|
verdictString := getVerdictString()
|
|
|
|
newVerdictObject := verdictObject{
|
|
Verdict: verdictString,
|
|
VerdictTime: currentTime,
|
|
}
|
|
|
|
identityVerdictHistoryMapMutex.Lock()
|
|
|
|
identityVerdictObjectsList, exists := identityVerdictHistoryMap[identityHash]
|
|
if (exists == false){
|
|
identityVerdictHistoryMap[identityHash] = []verdictObject{newVerdictObject}
|
|
} else {
|
|
identityVerdictObjectsList = append(identityVerdictObjectsList, newVerdictObject)
|
|
identityVerdictHistoryMap[identityHash] = identityVerdictObjectsList
|
|
}
|
|
|
|
identityVerdictHistoryMapMutex.Unlock()
|
|
|
|
identityVerdictLastRecordedMapMutex.Lock()
|
|
identityVerdictLastRecordedMap[identityHash] = currentTime
|
|
identityVerdictLastRecordedMapMutex.Unlock()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// This is run automatically by backgroundJobs on an internal
|
|
func RecordProfileVerdictsToHistoryMap(networkType byte)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("RecordProfileVerdictsToHistoryMap called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
// We use a map to avoid duplicates
|
|
allUserProfileHashesMap := make(map[[28]byte]struct{})
|
|
|
|
allProfileHashesWithMetadataList, err := badgerDatabase.GetAllProfileHashesWithMetadata()
|
|
if (err != nil) { return err }
|
|
|
|
for _, profileHash := range allProfileHashesWithMetadataList{
|
|
allUserProfileHashesMap[profileHash] = struct{}{}
|
|
}
|
|
|
|
allDownloadedProfileHashes, err := profileStorage.GetAllStoredProfileHashes()
|
|
if (err != nil) { return err }
|
|
|
|
for _, profileHash := range allDownloadedProfileHashes{
|
|
allUserProfileHashesMap[profileHash] = struct{}{}
|
|
}
|
|
|
|
allReviewedProfileHashesList, err := badgerDatabase.GetAllReviewedProfileHashes()
|
|
if (err != nil) { return err }
|
|
|
|
for _, profileHash := range allReviewedProfileHashesList{
|
|
allUserProfileHashesMap[profileHash] = struct{}{}
|
|
}
|
|
|
|
for profileHash, _ := range allUserProfileHashesMap{
|
|
|
|
currentTime := time.Now().Unix()
|
|
|
|
profileVerdictLastRecordedMapMutex.RLock()
|
|
verdictLastRecordedTime, exists := profileVerdictLastRecordedMap[profileHash]
|
|
profileVerdictLastRecordedMapMutex.RUnlock()
|
|
if (exists == true){
|
|
|
|
timeElapsed := currentTime - verdictLastRecordedTime
|
|
if (timeElapsed < VerdictIntervalDuration){
|
|
// We have recorded this consensus verdict recently enough. Skip to the next profile
|
|
continue
|
|
}
|
|
}
|
|
|
|
profileIsDisabled, profileMetadataIsKnown, _, profileNetworkType, _, _, downloadingRequiredReviews, parametersExist, profileConsensusVerdict, _, _, _, _, _, _, _, _, _, _, _, _, _, err := verifiedVerdict.GetVerifiedProfileVerdict(profileHash)
|
|
if (err != nil) { return err }
|
|
if (profileIsDisabled == true){
|
|
// Profile is disabled, it cannot be reviewed.
|
|
continue
|
|
}
|
|
if (profileMetadataIsKnown == false){
|
|
// Metadata must have been deleted after earlier check
|
|
// We cannot determine verified verdict.
|
|
continue
|
|
}
|
|
if (profileNetworkType != networkType){
|
|
// This profile belongs to a different network type
|
|
// We don't store its verdict in the verdict history
|
|
continue
|
|
}
|
|
if (downloadingRequiredReviews == false || parametersExist == false){
|
|
// We do not know the consensus verdict
|
|
continue
|
|
}
|
|
|
|
newVerdictObject := verdictObject{
|
|
Verdict: profileConsensusVerdict,
|
|
VerdictTime: currentTime,
|
|
}
|
|
|
|
profileVerdictHistoryMapMutex.Lock()
|
|
|
|
profileVerdictObjectsList, exists := profileVerdictHistoryMap[profileHash]
|
|
if (exists == false){
|
|
profileVerdictHistoryMap[profileHash] = []verdictObject{newVerdictObject}
|
|
} else {
|
|
profileVerdictObjectsList = append(profileVerdictObjectsList, newVerdictObject)
|
|
profileVerdictHistoryMap[profileHash] = profileVerdictObjectsList
|
|
}
|
|
|
|
profileVerdictHistoryMapMutex.Unlock()
|
|
|
|
profileVerdictLastRecordedMapMutex.Lock()
|
|
profileVerdictLastRecordedMap[profileHash] = currentTime
|
|
profileVerdictLastRecordedMapMutex.Unlock()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// This is run automatically by backgroundJobs on an internal
|
|
func RecordMessageVerdictsToHistoryMap(networkType byte)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("RecordMessageVerdictsToHistoryMap called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
allDownloadedMessageHashes, err := badgerDatabase.GetAllChatMessageHashes()
|
|
if (err != nil) { return err }
|
|
|
|
allReviewedMessageHashes, err := badgerDatabase.GetAllReviewedMessageHashes()
|
|
if (err != nil) { return err }
|
|
|
|
allMessageHashesList := helpers.CombineTwoListsAndAvoidDuplicates(allDownloadedMessageHashes, allReviewedMessageHashes)
|
|
|
|
for _, messageHash := range allMessageHashesList{
|
|
|
|
currentTime := time.Now().Unix()
|
|
|
|
messageVerdictLastRecordedMapMutex.RLock()
|
|
verdictLastRecordedTime, exists := messageVerdictLastRecordedMap[messageHash]
|
|
messageVerdictLastRecordedMapMutex.RUnlock()
|
|
if (exists == true){
|
|
|
|
timeElapsed := currentTime - verdictLastRecordedTime
|
|
if (timeElapsed < VerdictIntervalDuration){
|
|
// We have recorded this verdict recently enough. Skip to the next message.
|
|
continue
|
|
}
|
|
}
|
|
|
|
messageMetadataIsKnown, _, messageNetworkType, _, _, downloadingRequiredReviews, parametersExist, messageVerdict, _, _, _, _, _, _, err := verifiedVerdict.GetVerifiedMessageVerdict(messageHash)
|
|
if (err != nil) { return err }
|
|
if (messageMetadataIsKnown == false){
|
|
// Message must have been deleted from database after we retrieved the message hashes list
|
|
continue
|
|
}
|
|
if (messageNetworkType != networkType){
|
|
// The message belongs to a different network type
|
|
// We will not store its verdict in the verdict history
|
|
continue
|
|
}
|
|
if (downloadingRequiredReviews == false || parametersExist == false){
|
|
// We do not know sticky consensus verdict
|
|
continue
|
|
}
|
|
|
|
newVerdictObject := verdictObject{
|
|
Verdict: messageVerdict,
|
|
VerdictTime: currentTime,
|
|
}
|
|
|
|
messageVerdictHistoryMapMutex.Lock()
|
|
|
|
messageVerdictObjectsList, exists := messageVerdictHistoryMap[messageHash]
|
|
if (exists == false){
|
|
messageVerdictHistoryMap[messageHash] = []verdictObject{newVerdictObject}
|
|
} else {
|
|
messageVerdictObjectsList = append(messageVerdictObjectsList, newVerdictObject)
|
|
messageVerdictHistoryMap[messageHash] = messageVerdictObjectsList
|
|
}
|
|
|
|
messageVerdictHistoryMapMutex.Unlock()
|
|
|
|
messageVerdictLastRecordedMapMutex.Lock()
|
|
messageVerdictLastRecordedMap[messageHash] = currentTime
|
|
messageVerdictLastRecordedMapMutex.Unlock()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
|
|
|