// verdictHistory provides functions to keep track of and retrieve the verdict history for messages, profiles, and identities package verdictHistory // Verdict histories are used for 2 things: // 1. Sticky consensus statuses are calculated using the verdict history of a reviewedHash // 2. Hosts use the verdict history of a reviewedHash to know when to delete banned content // Once a profile/identity/message has been banned for a long enough time period, the host's client will delete the content //TODO: Add job to prune verdicts we no longer need //TODO: Store the verdict history as messagePack files, so if a user briefly closes client, history is maintained upon restart import "seekia/internal/badgerDatabase" import "seekia/internal/helpers" import "seekia/internal/moderation/verifiedVerdict" import "seekia/internal/profiles/profileStorage" import "errors" import "time" import "sync" // Map Structure: Identity Hash -> Last time the verdict was recorded for this identityHash in the identityVerdictHistoryMap var identityVerdictLastRecordedMap map[[16]byte]int64 = make(map[[16]byte]int64) var identityVerdictLastRecordedMapMutex sync.RWMutex // Map Structure: Profile Hash -> Last time the verdict was recorded for this profileHash in the profileVerdictHistoryMap var profileVerdictLastRecordedMap map[[28]byte]int64 = make(map[[28]byte]int64) var profileVerdictLastRecordedMapMutex sync.RWMutex // Map Structure: Message Hash -> Last time the verdict was recorded for this messageHash in the messageVerdictHistoryMap var messageVerdictLastRecordedMap map[[26]byte]int64 = make(map[[26]byte]int64) var messageVerdictLastRecordedMapMutex sync.RWMutex type verdictObject struct{ // Consensus verdict ("Approved"/"Banned"/"Undecided"/"Not Banned") Verdict string VerdictTime int64 } // Map Structure: Identity hash -> List of verdict objects var identityVerdictHistoryMap map[[16]byte][]verdictObject = make(map[[16]byte][]verdictObject) var identityVerdictHistoryMapMutex sync.RWMutex // Map Structure: Profile hash -> List of verdict objects var profileVerdictHistoryMap map[[28]byte][]verdictObject = make(map[[28]byte][]verdictObject) var profileVerdictHistoryMapMutex sync.RWMutex // Map Structure: Message hash -> List of verdict objects var messageVerdictHistoryMap map[[26]byte][]verdictObject = make(map[[26]byte][]verdictObject) var messageVerdictHistoryMapMutex sync.RWMutex // This is the minimum duration in seconds between recording the consensus verdicts for each reviewedHash const VerdictIntervalDuration int64 = 180 // This function should be called whenever we switch network types func DeleteVerdictHistory(){ identityVerdictLastRecordedMapMutex.Lock() clear(identityVerdictLastRecordedMap) identityVerdictLastRecordedMapMutex.Unlock() profileVerdictLastRecordedMapMutex.Lock() clear(profileVerdictLastRecordedMap) profileVerdictLastRecordedMapMutex.Unlock() messageVerdictLastRecordedMapMutex.Lock() clear(messageVerdictLastRecordedMap) messageVerdictLastRecordedMapMutex.Unlock() identityVerdictHistoryMapMutex.Lock() clear(identityVerdictHistoryMap) identityVerdictHistoryMapMutex.Unlock() profileVerdictHistoryMapMutex.Lock() clear(profileVerdictHistoryMap) profileVerdictHistoryMapMutex.Unlock() messageVerdictHistoryMapMutex.Lock() clear(messageVerdictHistoryMap) messageVerdictHistoryMapMutex.Unlock() } //Outputs: // -[][16]byte: Reviewed identity hashes list // -error func GetAllReviewedIdentityHashesList()([][16]byte, error){ identityVerdictLastRecordedMapMutex.RLock() allReviewedIdentityHashesList := helpers.GetListOfMapKeys(identityVerdictLastRecordedMap) identityVerdictLastRecordedMapMutex.RUnlock() return allReviewedIdentityHashesList, nil } //Outputs: // -[][28]byte: Reviewed profile hashes list // -error func GetAllReviewedProfileHashesList()([][28]byte, error){ profileVerdictLastRecordedMapMutex.RLock() allReviewedProfileHashesList := helpers.GetListOfMapKeys(profileVerdictLastRecordedMap) profileVerdictLastRecordedMapMutex.RUnlock() return allReviewedProfileHashesList, nil } //Outputs: // -[][26]byte: Reviewed message hashes list // -error func GetAllReviewedMessageHashesList()([][26]byte, error){ messageVerdictLastRecordedMapMutex.RLock() allReviewedMessageHashesList := helpers.GetListOfMapKeys(messageVerdictLastRecordedMap) messageVerdictLastRecordedMapMutex.RUnlock() return allReviewedMessageHashesList, nil } //Outputs: // -map[int64]string: Verdict history map // -error func GetIdentityVerdictHistoryMap(identityHash [16]byte)(map[int64]string, error){ identityVerdictHistoryMapMutex.RLock() defer identityVerdictHistoryMapMutex.RUnlock() verdictObjectsList, exists := identityVerdictHistoryMap[identityHash] if (exists == false){ emptyMap := make(map[int64]string) return emptyMap, nil } verdictHistoryMap, err := getReviewedHashVerdictHistoryMap(verdictObjectsList) if (err != nil) { return nil, err } return verdictHistoryMap, nil } //Outputs: // -map[int64]string: Verdict history map // -error func GetProfileVerdictHistoryMap(profileHash [28]byte)(map[int64]string, error){ profileVerdictHistoryMapMutex.RLock() defer profileVerdictHistoryMapMutex.RUnlock() verdictObjectsList, exists := profileVerdictHistoryMap[profileHash] if (exists == false){ emptyMap := make(map[int64]string) return emptyMap, nil } verdictHistoryMap, err := getReviewedHashVerdictHistoryMap(verdictObjectsList) if (err != nil) { return nil, err } return verdictHistoryMap, nil } //Outputs: // -map[int64]string: Verdict history map // -error func GetMessageVerdictHistoryMap(messageHash [26]byte)(map[int64]string, error){ messageVerdictHistoryMapMutex.RLock() defer messageVerdictHistoryMapMutex.RUnlock() verdictObjectsList, exists := messageVerdictHistoryMap[messageHash] if (exists == false){ emptyMap := make(map[int64]string) return emptyMap, nil } verdictHistoryMap, err := getReviewedHashVerdictHistoryMap(verdictObjectsList) if (err != nil) { return nil, err } return verdictHistoryMap, nil } func getReviewedHashVerdictHistoryMap(verdictObjectsList []verdictObject)(map[int64]string, error){ // Map Structure: Unix Time -> Verdict of reviewedHash at unix time reviewedHashVerdictHistoryMap := make(map[int64]string) for _, verdictObject := range verdictObjectsList{ verdict := verdictObject.Verdict verdictTime := verdictObject.VerdictTime reviewedHashVerdictHistoryMap[verdictTime] = verdict } return reviewedHashVerdictHistoryMap, nil } // This is run automatically by backgroundJobs on an internal func RecordIdentityVerdictsToHistoryMap(networkType byte)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("RecordIdentityVerdictsToHistoryMap called with invalid networkType: " + networkTypeString) } allDownloadedProfileIdentityHashes, err := profileStorage.GetAllStoredProfileIdentityHashes() if (err != nil) { return err } allReviewedIdentityHashes, err := badgerDatabase.GetAllReviewedIdentityHashes() if (err != nil) { return err } allUserIdentityHashesList := helpers.CombineTwoListsAndAvoidDuplicates(allDownloadedProfileIdentityHashes, allReviewedIdentityHashes) for _, identityHash := range allUserIdentityHashesList{ currentTime := time.Now().Unix() identityVerdictLastRecordedMapMutex.RLock() verdictLastRecordedTime, exists := identityVerdictLastRecordedMap[identityHash] identityVerdictLastRecordedMapMutex.RUnlock() if (exists == true){ timeElapsed := currentTime - verdictLastRecordedTime if (timeElapsed < VerdictIntervalDuration){ // We have recorded this consensus verdict recently enough. // We go to the next identity continue } } downloadingRequiredReviews, parametersExist, identityIsBanned, _, _, _, err := verifiedVerdict.GetVerifiedIdentityVerdict(identityHash, networkType) if (err != nil) { return err } if (downloadingRequiredReviews == false || parametersExist == false){ // We do not know consensus verdict // We go to the next identity continue } getVerdictString := func()string{ if (identityIsBanned == true){ return "Banned" } return "Not Banned" } verdictString := getVerdictString() newVerdictObject := verdictObject{ Verdict: verdictString, VerdictTime: currentTime, } identityVerdictHistoryMapMutex.Lock() identityVerdictObjectsList, exists := identityVerdictHistoryMap[identityHash] if (exists == false){ identityVerdictHistoryMap[identityHash] = []verdictObject{newVerdictObject} } else { identityVerdictObjectsList = append(identityVerdictObjectsList, newVerdictObject) identityVerdictHistoryMap[identityHash] = identityVerdictObjectsList } identityVerdictHistoryMapMutex.Unlock() identityVerdictLastRecordedMapMutex.Lock() identityVerdictLastRecordedMap[identityHash] = currentTime identityVerdictLastRecordedMapMutex.Unlock() } return nil } // This is run automatically by backgroundJobs on an internal func RecordProfileVerdictsToHistoryMap(networkType byte)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("RecordProfileVerdictsToHistoryMap called with invalid networkType: " + networkTypeString) } // We use a map to avoid duplicates allUserProfileHashesMap := make(map[[28]byte]struct{}) allProfileHashesWithMetadataList, err := badgerDatabase.GetAllProfileHashesWithMetadata() if (err != nil) { return err } for _, profileHash := range allProfileHashesWithMetadataList{ allUserProfileHashesMap[profileHash] = struct{}{} } allDownloadedProfileHashes, err := profileStorage.GetAllStoredProfileHashes() if (err != nil) { return err } for _, profileHash := range allDownloadedProfileHashes{ allUserProfileHashesMap[profileHash] = struct{}{} } allReviewedProfileHashesList, err := badgerDatabase.GetAllReviewedProfileHashes() if (err != nil) { return err } for _, profileHash := range allReviewedProfileHashesList{ allUserProfileHashesMap[profileHash] = struct{}{} } for profileHash, _ := range allUserProfileHashesMap{ currentTime := time.Now().Unix() profileVerdictLastRecordedMapMutex.RLock() verdictLastRecordedTime, exists := profileVerdictLastRecordedMap[profileHash] profileVerdictLastRecordedMapMutex.RUnlock() if (exists == true){ timeElapsed := currentTime - verdictLastRecordedTime if (timeElapsed < VerdictIntervalDuration){ // We have recorded this consensus verdict recently enough. Skip to the next profile continue } } profileIsDisabled, profileMetadataIsKnown, _, profileNetworkType, _, _, downloadingRequiredReviews, parametersExist, profileConsensusVerdict, _, _, _, _, _, _, _, _, _, _, _, _, _, err := verifiedVerdict.GetVerifiedProfileVerdict(profileHash) if (err != nil) { return err } if (profileIsDisabled == true){ // Profile is disabled, it cannot be reviewed. continue } if (profileMetadataIsKnown == false){ // Metadata must have been deleted after earlier check // We cannot determine verified verdict. continue } if (profileNetworkType != networkType){ // This profile belongs to a different network type // We don't store its verdict in the verdict history continue } if (downloadingRequiredReviews == false || parametersExist == false){ // We do not know the consensus verdict continue } newVerdictObject := verdictObject{ Verdict: profileConsensusVerdict, VerdictTime: currentTime, } profileVerdictHistoryMapMutex.Lock() profileVerdictObjectsList, exists := profileVerdictHistoryMap[profileHash] if (exists == false){ profileVerdictHistoryMap[profileHash] = []verdictObject{newVerdictObject} } else { profileVerdictObjectsList = append(profileVerdictObjectsList, newVerdictObject) profileVerdictHistoryMap[profileHash] = profileVerdictObjectsList } profileVerdictHistoryMapMutex.Unlock() profileVerdictLastRecordedMapMutex.Lock() profileVerdictLastRecordedMap[profileHash] = currentTime profileVerdictLastRecordedMapMutex.Unlock() } return nil } // This is run automatically by backgroundJobs on an internal func RecordMessageVerdictsToHistoryMap(networkType byte)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("RecordMessageVerdictsToHistoryMap called with invalid networkType: " + networkTypeString) } allDownloadedMessageHashes, err := badgerDatabase.GetAllChatMessageHashes() if (err != nil) { return err } allReviewedMessageHashes, err := badgerDatabase.GetAllReviewedMessageHashes() if (err != nil) { return err } allMessageHashesList := helpers.CombineTwoListsAndAvoidDuplicates(allDownloadedMessageHashes, allReviewedMessageHashes) for _, messageHash := range allMessageHashesList{ currentTime := time.Now().Unix() messageVerdictLastRecordedMapMutex.RLock() verdictLastRecordedTime, exists := messageVerdictLastRecordedMap[messageHash] messageVerdictLastRecordedMapMutex.RUnlock() if (exists == true){ timeElapsed := currentTime - verdictLastRecordedTime if (timeElapsed < VerdictIntervalDuration){ // We have recorded this verdict recently enough. Skip to the next message. continue } } messageMetadataIsKnown, _, messageNetworkType, _, _, downloadingRequiredReviews, parametersExist, messageVerdict, _, _, _, _, _, _, err := verifiedVerdict.GetVerifiedMessageVerdict(messageHash) if (err != nil) { return err } if (messageMetadataIsKnown == false){ // Message must have been deleted from database after we retrieved the message hashes list continue } if (messageNetworkType != networkType){ // The message belongs to a different network type // We will not store its verdict in the verdict history continue } if (downloadingRequiredReviews == false || parametersExist == false){ // We do not know sticky consensus verdict continue } newVerdictObject := verdictObject{ Verdict: messageVerdict, VerdictTime: currentTime, } messageVerdictHistoryMapMutex.Lock() messageVerdictObjectsList, exists := messageVerdictHistoryMap[messageHash] if (exists == false){ messageVerdictHistoryMap[messageHash] = []verdictObject{newVerdictObject} } else { messageVerdictObjectsList = append(messageVerdictObjectsList, newVerdictObject) messageVerdictHistoryMap[messageHash] = messageVerdictObjectsList } messageVerdictHistoryMapMutex.Unlock() messageVerdictLastRecordedMapMutex.Lock() messageVerdictLastRecordedMap[messageHash] = currentTime messageVerdictLastRecordedMapMutex.Unlock() } return nil }