2024-04-11 15:51:56 +02:00
// queryHosts provides functions to download information from hosts
// These include parameters, profiles, messages, reports, reviews, identity deposits, and trusted moderation viewable statuses
// Each function will query a specified number of eligible hosts
package queryHosts
//TODO: If you download a user's identity, then the host is saying that the user's identity is funded. Add it to trustedFundedStatus
//TODO: Split requests up so that memory does not have to store all of the requested hashes
//TODO: Set a maximum limit, so that only a certain amount of data will be requested from each host
//TODO: Choose hosts more intelligently based on what ranges we are requesting
// Right now, we are sending requests to randomly chosen hosts until we reach the numberOfHostsToQuery
// This is a bad strategy, because, if we are requesting range 20-100, we might request from a host that is only hosting from range 10-25
// We should use numberOfHostsToQuery to choose a set of hosts that will host the entire range
// We must make sure that we sometimes request from hosts that only host a small number of total profiles.
// TODO: Add BroadcastToHosts, GetFundedStatuses, and more
//TODO: If only Host/Moderator mode is enabled, we can skip all of the downloaded identity/inbox fingerprinting prevention logic
// This logic is designed to prevent hosts from learning the user's moderator/host identity/behavior from their downloaded content
// For example, a user is moderating and hosting. requestors could determine a host's moderator identity by querying for
// profiles outside of their host range and seeing which mate profiles they have downloaded.
// Using this information, they could learn their moderator range and link it to their moderator identity/profile.
//
// We cannot disable fingerprinting if Mate mode is enabled, because a user's Mate criteria will change, so we cannot allow the host to know what their criteria was previously
// This is also true if a Host or Moderator's ranges change, but in those cases, the leak of privacy is fine
// This is because the information (previous host/moderator ranges) is not sensitive, and not a problem if it is leaked
import "seekia/internal/byteRange"
import "seekia/internal/badgerDatabase"
import "seekia/internal/contentMetadata"
import "seekia/internal/encoding"
import "seekia/internal/helpers"
import "seekia/internal/identity"
import "seekia/internal/logger"
import "seekia/internal/messaging/chatMessageStorage"
import "seekia/internal/moderation/moderatorScores"
import "seekia/internal/moderation/readReports"
import "seekia/internal/moderation/readReviews"
import "seekia/internal/moderation/reportStorage"
import "seekia/internal/moderation/reviewStorage"
import "seekia/internal/moderation/trustedAddressDeposits"
import "seekia/internal/moderation/trustedViewableStatus"
import "seekia/internal/network/eligibleHosts"
import "seekia/internal/network/hostRanges"
import "seekia/internal/network/maliciousHosts"
import "seekia/internal/network/mateCriteria"
import "seekia/internal/network/peerClient"
import "seekia/internal/network/sendRequests"
import "seekia/internal/network/serverResponse"
import "seekia/internal/parameters/parametersStorage"
import "seekia/internal/profiles/profileStorage"
import "seekia/internal/profiles/readProfiles"
import "bytes"
import "errors"
import "slices"
func DownloadParametersFromHosts ( allowClearnet bool , networkType byte , numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadParametersFromHosts called with invalid networkType: " + networkTypeString )
}
// Outputs:
// -bool: Successful download from host
// -error
downloadParametersFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
exists , hostIsHostingParameters , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "HostingParameters" )
if ( err != nil ) { return false , err }
if ( exists == false ) {
return false , errors . New ( "Database corrupt: Contains host Profile missing HostingParameters" )
}
if ( hostIsHostingParameters != "Yes" ) {
// This host is not hosting parameters, skip
return false , nil
}
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( hostProfileExists == false ) {
return false , nil
}
if ( connectionEstablished == false ) {
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
downloadSuccessful , parametersInfoMap , err := sendRequests . GetParametersInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
return false , nil
}
if ( len ( parametersInfoMap ) == 0 ) {
// The host has provided no parameters. We will not consider this a successful download
return false , nil
}
2024-06-11 06:59:06 +02:00
// We will check to see if our parameters creation time is older than the one the host is offering
2024-04-11 15:51:56 +02:00
// If it is, we will download it and continue
parametersTypesToRetrieveList := make ( [ ] string , 0 )
2024-06-11 06:59:06 +02:00
for parametersType , parametersCreationTime := range parametersInfoMap {
2024-04-11 15:51:56 +02:00
2024-06-11 06:59:06 +02:00
storedParametersFound , _ , _ , storedParametersCreationTime , err := parametersStorage . GetAuthorizedParameters ( parametersType , networkType )
2024-04-11 15:51:56 +02:00
if ( err != nil ) { return false , err }
if ( storedParametersFound == true ) {
2024-06-11 06:59:06 +02:00
if ( storedParametersCreationTime < parametersCreationTime ) {
2024-04-11 15:51:56 +02:00
// The offered parameters are not newer than our existing parameters
// Skip downloading them.
continue
}
}
parametersTypesToRetrieveList = append ( parametersTypesToRetrieveList , parametersType )
}
if ( len ( parametersTypesToRetrieveList ) == 0 ) {
// The host has no parameters that we need. Nothing to download.
return true , nil
}
// Now we download parameters
downloadSuccessful , receivedParametersList , err := sendRequests . GetParametersFromHost ( connectionIdentifier , hostIdentityHash , networkType , parametersTypesToRetrieveList )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Unable to download parameters. Nothing left to do with this host.
return false , nil
}
if ( len ( receivedParametersList ) == 0 ) {
// Host is probably malicious
// Host could have received new admin permissions after we got parameters info
// Either way, we do not count this as a successful download
return false , nil
}
for _ , parametersBytes := range receivedParametersList {
parametersAreValid , _ , err := parametersStorage . AddParameters ( parametersBytes )
if ( err != nil ) { return false , err }
if ( parametersAreValid == false ) {
return false , errors . New ( "GetParametersFromHost not verifying parameters." )
}
}
// These parameters were downloaded, not necessarily authorized or added
numberOfDownloadedParametersString := helpers . ConvertIntToString ( len ( receivedParametersList ) )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfDownloadedParametersString + " network parameters from host." )
if ( err != nil ) { return false , err }
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadParametersFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadProfilesFromHosts (
allowClearnet bool ,
networkType byte ,
profileTypeToRetrieve string ,
rangeToRetrieveStart [ 16 ] byte ,
rangeToRetrieveEnd [ 16 ] byte ,
identityHashesToRetrieveList [ ] [ 16 ] byte ,
criteria [ ] byte ,
getNewestProfilesOnly bool ,
getViewableProfilesOnly bool ,
downloadAllMode bool ,
checkIfProfileShouldBeDownloaded func ( [ 28 ] byte , [ 16 ] byte , int64 ) ( bool , error ) ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadProfilesFromHosts called with invalid networkType: " + networkTypeString )
}
// We cycle through each host
// We get the profiles info for profiles that are within our requested criteria/range/identitites list
// We see if any are knowingly outside of criteria, and download anyway if they are (to resist fingerprinting)
// We download the profiles.
// We don't download any profiles that we already have (if desiresPruningMode == false)
//TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable profiles
//Outputs:
// -bool: Successfully downloaded from host (false if host does not have any profiles we need)
// -error
downloadProfilesFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
hostIsHostingProfileType , theirHostRangeStart , theirHostRangeEnd , err := hostRanges . GetHostedIdentityHashRangeFromHostRawProfileMap ( hostRawProfileMap , profileTypeToRetrieve )
if ( err != nil ) { return false , err }
if ( hostIsHostingProfileType == false ) {
// Host is not hosting any profiles of this profileType. Skip to next host
return false , nil
}
//Outputs:
// -bool: Download successful
// -[21]byte: Connection identifier
// -[]serverResponse.ProfileInfoStruct: Retrieved profiles info objects list
// -error
getRetrievedProfileInfoObjectsList := func ( ) ( bool , [ 21 ] byte , [ ] serverResponse . ProfileInfoStruct , error ) {
// Our request will either be based on identity hash range or identity hash list
// We will split up the request into either ranges or identity hashes
maximumProfilesInGetProfilesInfoRequest := serverResponse . MaximumProfilesInResponse_GetProfilesInfo - 100
if ( len ( identityHashesToRetrieveList ) != 0 ) {
// We will retrieve identities lists, not identity ranges
anyValuesExist , identitiesInMyRangeList , err := byteRange . GetAllIdentityHashesInListWithinRange ( rangeToRetrieveStart , rangeToRetrieveEnd , identityHashesToRetrieveList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( anyValuesExist == false ) {
// This should not happen
return false , [ 21 ] byte { } , nil , errors . New ( "Identity hashes list to retrieve contains no identities within request range." )
}
anyValuesExist , identitiesInTheirRangeList , err := byteRange . GetAllIdentityHashesInListWithinRange ( theirHostRangeStart , theirHostRangeEnd , identitiesInMyRangeList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( anyValuesExist == false ) {
// This host is not hosting any identities we desire. We will skip them.
return false , [ 21 ] byte { } , nil , nil
}
// Now we split request into sublists if needed
getMaximumIdentitiesPerRequest := func ( ) int {
if ( getNewestProfilesOnly == true ) {
return maximumProfilesInGetProfilesInfoRequest
}
// We assume each identity has approximately 3 profiles
maximumIdentitiesPerRequest := maximumProfilesInGetProfilesInfoRequest / 3
return maximumIdentitiesPerRequest
}
maximumIdentitiesPerRequest := getMaximumIdentitiesPerRequest ( )
sublistsToQueryList , err := helpers . SplitListIntoSublists ( identitiesInTheirRangeList , maximumIdentitiesPerRequest )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , nil
}
downloadProfileInfoObjectsList := func ( ) ( bool , [ ] serverResponse . ProfileInfoStruct , error ) {
retrievedProfileInfoObjectsList := make ( [ ] serverResponse . ProfileInfoStruct , 0 )
minimumRange , maximumRange := byteRange . GetMinimumMaximumIdentityHashBounds ( )
for _ , identityHashesSublist := range sublistsToQueryList {
downloadSuccessful , profilesInfoObjectsList , err := sendRequests . GetProfilesInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , profileTypeToRetrieve , minimumRange , maximumRange , identityHashesSublist , criteria , getNewestProfilesOnly , getViewableProfilesOnly )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this subrange.
// Could be that host went offline permanently or just this connection failed.
// Either way, cease connection to host
err := logger . AddLogEntry ( "Network" , "Failed to download profiles info from host." )
if ( err != nil ) { return false , nil , err }
err = peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
retrievedProfileInfoObjectsList = append ( retrievedProfileInfoObjectsList , profilesInfoObjectsList ... )
}
return true , retrievedProfileInfoObjectsList , nil
}
downloadSuccessful , retrievedProfileInfoObjectsList , err := downloadProfileInfoObjectsList ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err = peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedProfileInfoObjectsList , nil
}
// We will retrieve identity ranges, not identity lists
//Outputs:
// -Any intersection found
// -[16]byte: Intersection range start
// -[16]byte: Intersection range end
// -error
getIntersectionRange := func ( ) ( bool , [ 16 ] byte , [ 16 ] byte , error ) {
if ( profileTypeToRetrieve != "Mate" ) {
// Hosts must host all or none of host/moderator identities, so intersection is always all our requested identities
return true , rangeToRetrieveStart , rangeToRetrieveEnd , nil
}
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := byteRange . GetIdentityIntersectionRangeFromTwoRanges ( rangeToRetrieveStart , rangeToRetrieveEnd , theirHostRangeStart , theirHostRangeEnd )
if ( err != nil ) { return false , [ 16 ] byte { } , [ 16 ] byte { } , err }
if ( anyIntersectionFound == false ) {
return false , [ 16 ] byte { } , [ 16 ] byte { } , nil
}
return true , intersectionRangeStart , intersectionRangeEnd , nil
}
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := getIntersectionRange ( )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( anyIntersectionFound == false ) {
// Host is not hosting any profiles within the range that we are requesting
// Skip this host
return false , [ 21 ] byte { } , nil , nil
}
exists , hostedProfilesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , profileTypeToRetrieve + "ProfilesQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile missing profilesQuantity" )
}
hostedProfilesQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedProfilesQuantity )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( hostedProfilesQuantityInt64 == 0 ) {
// Host must have just started their node. Skip them.
return false , [ 21 ] byte { } , nil , nil
}
estimatedNumItemsInSubrange , err := byteRange . GetEstimatedIdentitySubrangeQuantity ( theirHostRangeStart , theirHostRangeEnd , hostedProfilesQuantityInt64 , intersectionRangeStart , intersectionRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
subrangesToQueryList , err := byteRange . SplitIdentityRangeIntoEqualSubranges ( intersectionRangeStart , intersectionRangeEnd , estimatedNumItemsInSubrange , int64 ( maximumProfilesInGetProfilesInfoRequest ) )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , nil
}
//Outputs:
// -bool: Download successful
// -[]serverResponse.ProfileInfoStruct: Retrieved profiles info map list
// -error
downloadProfileInfoObjectsList := func ( ) ( bool , [ ] serverResponse . ProfileInfoStruct , error ) {
retrievedProfileInfoObjectsList := make ( [ ] serverResponse . ProfileInfoStruct , 0 )
for _ , subrangeObject := range subrangesToQueryList {
subrangeStart := subrangeObject . SubrangeStart
subrangeEnd := subrangeObject . SubrangeEnd
downloadSuccessful , profileInfoObjectsList , err := sendRequests . GetProfilesInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , profileTypeToRetrieve , subrangeStart , subrangeEnd , identityHashesToRetrieveList , criteria , getNewestProfilesOnly , getViewableProfilesOnly )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this subrange.
// Could be that host went offline permanently or just this connection failed.
// Either way, close connection
err := logger . AddLogEntry ( "Network" , "Failed to download profiles info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
retrievedProfileInfoObjectsList = append ( retrievedProfileInfoObjectsList , profileInfoObjectsList ... )
}
return true , retrievedProfileInfoObjectsList , nil
}
downloadSuccessful , retrievedProfileInfoObjectsList , err := downloadProfileInfoObjectsList ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err = peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedProfileInfoObjectsList , nil
}
downloadSuccessful , connectionIdentifier , retrievedProfilesInfoObjectsList , err := getRetrievedProfileInfoObjectsList ( )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
if ( len ( retrievedProfilesInfoObjectsList ) == 0 ) {
// Host has no profiles that we need. We are done with this host.
return true , nil
}
profileHashesToDownloadList := make ( [ ] [ 28 ] byte , 0 )
2024-06-11 06:59:06 +02:00
// Structure: Profile Hash -> Expected Creation Time
profileCreationTimesMap := make ( map [ [ 28 ] byte ] int64 )
2024-04-11 15:51:56 +02:00
// Structure: Profile Hash -> Identity Hash
profileIdentityHashesMap := make ( map [ [ 28 ] byte ] [ 16 ] byte )
hostIsMalicious := false
for _ , profileInfoObject := range retrievedProfilesInfoObjectsList {
receivedProfileHash := profileInfoObject . ProfileHash
receivedProfileAuthor := profileInfoObject . ProfileAuthor
2024-06-11 06:59:06 +02:00
receivedProfileCreationTime := profileInfoObject . ProfileCreationTime
2024-04-11 15:51:56 +02:00
// Now we check to see if we already have the profile
// We check to see if it is known to be outside of requested range/identities list/criteria (host is malicious)
// If it is, we still request it in our download, and then set host as malicious afterwards
// Otherwise, host would know that we have this profile stored
//TODO: Disable this step if only host/moderator mode is enabled
storedProfileExists , storedProfileBytes , err := badgerDatabase . GetUserProfile ( profileTypeToRetrieve , receivedProfileHash )
if ( err != nil ) { return false , err }
checkIfHostIsOfferingInvalidProfile := func ( ) ( bool , error ) {
if ( storedProfileExists == false ) {
return false , nil
}
2024-06-11 06:59:06 +02:00
ableToRead , storedProfileHash , profileVersion , profileNetworkType , storedProfileAuthor , storedProfileCreationTime , _ , storedRawProfileMap , err := readProfiles . ReadProfileAndHash ( false , storedProfileBytes )
2024-04-11 15:51:56 +02:00
if ( err != nil ) { return false , err }
if ( ableToRead == false ) {
return false , errors . New ( "Database corrupt: Contains invalid profile." )
}
if ( storedProfileHash != receivedProfileHash ) {
return false , errors . New ( "Database corrupt: Contains profile with different hash than entry key." )
}
if ( profileNetworkType != networkType ) {
return true , nil
}
if ( storedProfileAuthor != receivedProfileAuthor ) {
return true , nil
}
2024-06-11 06:59:06 +02:00
if ( storedProfileCreationTime != receivedProfileCreationTime ) {
2024-04-11 15:51:56 +02:00
return true , nil
}
if ( criteria != nil ) {
criteriaIsValid , fulfillsCriteria , err := mateCriteria . CheckIfMateProfileFulfillsCriteria ( false , profileVersion , storedRawProfileMap , criteria )
if ( err != nil ) { return false , err }
if ( criteriaIsValid == false ) {
return false , errors . New ( "DownloadProfilesFromHosts called with invalid criteria." )
}
if ( fulfillsCriteria == false ) {
return true , nil
}
}
return false , nil
}
hostIsOfferingInvalidProfile , err := checkIfHostIsOfferingInvalidProfile ( )
if ( err != nil ) { return false , err }
if ( hostIsOfferingInvalidProfile == true ) {
// Host is malicious, we must still download the profile
hostIsMalicious = true
profileHashesToDownloadList = append ( profileHashesToDownloadList , receivedProfileHash )
profileIdentityHashesMap [ receivedProfileHash ] = receivedProfileAuthor
2024-06-11 06:59:06 +02:00
profileCreationTimesMap [ receivedProfileHash ] = receivedProfileCreationTime
2024-04-11 15:51:56 +02:00
continue
}
if ( downloadAllMode == true ) {
// downloadAllMode is enabled, we must download all profiles regardless of if we already have it
// This is done to prevent hosts learning our private desires
profileHashesToDownloadList = append ( profileHashesToDownloadList , receivedProfileHash )
profileIdentityHashesMap [ receivedProfileHash ] = receivedProfileAuthor
2024-06-11 06:59:06 +02:00
profileCreationTimesMap [ receivedProfileHash ] = receivedProfileCreationTime
2024-04-11 15:51:56 +02:00
continue
}
if ( storedProfileExists == true ) {
// We already have profile, skip downloading it
if ( getViewableProfilesOnly == true ) {
err = trustedViewableStatus . AddTrustedProfileIsViewableStatus ( receivedProfileHash , hostIdentityHash , true )
if ( err != nil ) { return false , err }
}
continue
}
// Profile does not exist, we must check if we want to download it
// This works differently for Hosting, Moderation, and Browsing
2024-06-11 06:59:06 +02:00
shouldDownload , err := checkIfProfileShouldBeDownloaded ( receivedProfileHash , receivedProfileAuthor , receivedProfileCreationTime )
2024-04-11 15:51:56 +02:00
if ( err != nil ) { return false , err }
if ( shouldDownload == true ) {
profileHashesToDownloadList = append ( profileHashesToDownloadList , receivedProfileHash )
profileIdentityHashesMap [ receivedProfileHash ] = receivedProfileAuthor
2024-06-11 06:59:06 +02:00
profileCreationTimesMap [ receivedProfileHash ] = receivedProfileCreationTime
2024-04-11 15:51:56 +02:00
continue
}
}
if ( len ( profileHashesToDownloadList ) == 0 ) {
// Nothing to download, done with this host.
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
return true , nil
}
// Now we retrieve profiles from host
maximumProfileHashesInResponse , err := serverResponse . GetMaximumProfilesInResponse_GetProfiles ( profileTypeToRetrieve )
if ( err != nil ) { return false , err }
profileHashesSublistsList , err := helpers . SplitListIntoSublists ( profileHashesToDownloadList , maximumProfileHashesInResponse )
if ( err != nil ) { return false , err }
numberOfDownloadedProfiles := 0
for _ , profileHashesSublist := range profileHashesSublistsList {
2024-06-11 06:59:06 +02:00
downloadSuccessful , profilesList , err := sendRequests . GetProfilesFromHost ( connectionIdentifier , hostIdentityHash , networkType , profileTypeToRetrieve , profileHashesSublist , profileIdentityHashesMap , profileCreationTimesMap , nil )
2024-04-11 15:51:56 +02:00
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Request has failed
// We will stop downloading from this host
break
}
for _ , profileBytes := range profilesList {
//TODO: If downloadAll mode is enabled, we need to check to see if profile should be imported or not (desires pruning mode)
profileIsWellFormed , addedProfileHash , err := profileStorage . AddUserProfile ( profileBytes )
if ( err != nil ) { return false , err }
if ( profileIsWellFormed == false ) {
return false , errors . New ( "GetProfilesFromHost not verifying profiles are well formed." )
}
if ( getViewableProfilesOnly == true ) {
err = trustedViewableStatus . AddTrustedProfileIsViewableStatus ( addedProfileHash , hostIdentityHash , true )
if ( err != nil ) { return false , err }
}
}
numberOfDownloadedProfiles += len ( profilesList )
}
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
if ( numberOfDownloadedProfiles != 0 ) {
numberOfProfilesDownloadedString := helpers . ConvertIntToString ( numberOfDownloadedProfiles )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfProfilesDownloadedString + " " + profileTypeToRetrieve + " profiles from host." )
if ( err != nil ) { return false , err }
}
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadProfilesFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadMessagesFromHosts (
allowClearnet bool ,
networkType byte ,
rangeToRetrieveStart [ 10 ] byte ,
rangeToRetrieveEnd [ 10 ] byte ,
inboxesToRetrieveList [ ] [ 10 ] byte ,
getViewableMessagesOnly bool ,
getDecryptableMessagesOnly bool ,
checkIfMessageShouldBeDownloaded func ( [ 26 ] byte ) ( bool , error ) ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadMessagesFromHosts called with invalid networkType: " + networkTypeString )
}
// We cycle through each host
// We get all message hashes that they host that are within our requested range/inboxes list
// We see if any are knowingly outside of inboxes list, and download anyway if they are (to resist fingerprinting)
// We download the messages that we don't already have
//TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable messages
//Outputs:
// -bool: Successfully downloaded from host (false if host does not have any messages we need)
// -error
downloadMessagesFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
hostIsHostingMessages , theirInboxRangeStart , theirInboxRangeEnd , err := hostRanges . GetHostedMessageInboxesRangeFromHostRawProfileMap ( hostRawProfileMap )
if ( hostIsHostingMessages == false ) {
// Host is not hosting any messages. Skip to next host
return false , nil
}
//Outputs:
// -bool: Download successful
// -[21]byte: Connection identifier
// -[][26]byte: Retrieved message hashes list
// -[10]byte: Inbox Range start of requested messages
// -[10]byte: Inbox range end of requested messages
// -[][10]byte: Requested inboxes list (nil if none were requested)
// -error
getReceivedMessageHashesList := func ( ) ( bool , [ 21 ] byte , [ ] [ 26 ] byte , [ 10 ] byte , [ 10 ] byte , [ ] [ 10 ] byte , error ) {
// Our request will either be based on inbox range or inbox list
// We will split up the request into either ranges or inboxes
exists , hostedMessagesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "MessagesQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile missing MessagesQuantity" )
}
hostedMessagesQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedMessagesQuantity )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
maximumMessageHashesInResponse := serverResponse . MaximumMessageHashesInResponse_GetMessageHashesList - 100
if ( len ( inboxesToRetrieveList ) != 0 ) {
// We will split request into inbox lists, not inbox ranges
anyValuesExist , inboxesInMyRangeList , err := byteRange . GetAllInboxesInListWithinRange ( rangeToRetrieveStart , rangeToRetrieveEnd , inboxesToRetrieveList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( anyValuesExist == false ) {
// This should not happen
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , errors . New ( "Inboxes list to retrieve contains no inboxes within request range." )
}
anyValuesExist , inboxesInTheirRangeList , err := byteRange . GetAllInboxesInListWithinRange ( theirInboxRangeStart , theirInboxRangeEnd , inboxesInMyRangeList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( anyValuesExist == false ) {
// This host is not hosting any inboxes we desire. We will skip them.
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
// Now we split request into sublists if needed
exists , hostedInboxesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "InboxesQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile missing InboxesQuantity" )
}
hostedInboxesQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedInboxesQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile with invalid InboxesQuantity: " + hostedInboxesQuantity )
}
if ( hostedInboxesQuantityInt64 == 0 ) {
// Host has no inboxes
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
estimatedNumMessagesPerInbox := hostedMessagesQuantityInt64 / hostedInboxesQuantityInt64
maximumInboxesPerRequest := int64 ( maximumMessageHashesInResponse ) / estimatedNumMessagesPerInbox
maximumInboxesPerRequestInt , err := helpers . ConvertInt64ToInt ( maximumInboxesPerRequest )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
sublistsToQueryList , err := helpers . SplitListIntoSublists ( inboxesInTheirRangeList , maximumInboxesPerRequestInt )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
minimumRange , maximumRange := byteRange . GetMinimumMaximumInboxBounds ( )
//Outputs:
// -bool: Download successful
// -[][26]byte: Message hashes list
// -error
getMessageHashesListFromHost := func ( ) ( bool , [ ] [ 26 ] byte , error ) {
retrievedMessageHashesList := make ( [ ] [ 26 ] byte , 0 )
for _ , inboxesSublist := range sublistsToQueryList {
downloadSuccessful , messageHashesList , err := sendRequests . GetMessageHashesListFromHost ( connectionIdentifier , hostIdentityHash , networkType , minimumRange , maximumRange , inboxesSublist , getViewableMessagesOnly , getDecryptableMessagesOnly )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// We are done with this host
err := logger . AddLogEntry ( "Network" , "Failed to download message hashes from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
retrievedMessageHashesList = append ( retrievedMessageHashesList , messageHashesList ... )
}
return true , retrievedMessageHashesList , nil
}
downloadSuccessful , retrievedMessageHashesList , err := getMessageHashesListFromHost ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedMessageHashesList , minimumRange , maximumRange , inboxesInTheirRangeList , nil
}
// We will retrieve using inbox ranges, not inbox lists
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := byteRange . GetInboxIntersectionRangeFromTwoRanges ( rangeToRetrieveStart , rangeToRetrieveEnd , theirInboxRangeStart , theirInboxRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( anyIntersectionFound == false ) {
// Host is not hosting any messages within the inboxes range that we are requesting
// Skip this host
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
estimatedNumItemsInIntersectionRange , err := byteRange . GetEstimatedInboxSubrangeQuantity ( theirInboxRangeStart , theirInboxRangeEnd , hostedMessagesQuantityInt64 , intersectionRangeStart , intersectionRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
subrangesToQueryList , err := byteRange . SplitInboxRangeIntoEqualSubranges ( intersectionRangeStart , intersectionRangeEnd , estimatedNumItemsInIntersectionRange , int64 ( maximumMessageHashesInResponse ) )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
getMessageHashesListFromHost := func ( ) ( bool , [ ] [ 26 ] byte , error ) {
retrievedMessageHashesList := make ( [ ] [ 26 ] byte , 0 )
for _ , subrangeObject := range subrangesToQueryList {
subrangeStart := subrangeObject . SubrangeStart
subrangeEnd := subrangeObject . SubrangeEnd
downloadSuccessful , messageHashesList , err := sendRequests . GetMessageHashesListFromHost ( connectionIdentifier , hostIdentityHash , networkType , subrangeStart , subrangeEnd , inboxesToRetrieveList , getViewableMessagesOnly , getDecryptableMessagesOnly )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this subrange.
//Could be that host went offline permanently or just this connection failed.
// Either way, end connection to host
err := logger . AddLogEntry ( "Network" , "Failed to download message hashes from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
retrievedMessageHashesList = append ( retrievedMessageHashesList , messageHashesList ... )
}
return true , retrievedMessageHashesList , nil
}
downloadSuccessful , retrievedMessageHashesList , err := getMessageHashesListFromHost ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil , nil
}
emptyList := make ( [ ] [ 10 ] byte , 0 )
return true , connectionIdentifier , retrievedMessageHashesList , intersectionRangeStart , intersectionRangeEnd , emptyList , nil
}
downloadSuccessful , connectionIdentifier , retrievedMessageHashesList , requestedInboxesRangeStart , requestedInboxesRangeEnd , requestedInboxesList , err := getReceivedMessageHashesList ( )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
// Now we see if we have any of these messages, and if they do not fulfill the request range/inboxes list
// If they do not fulfill the request, we know host is malicious
// We still download the messages to prevent host from knowing that we have those messages downloaded
// We then mark the host as malicious
//TODO: Immediately stop connection to host if only Host/Moderator mode is enabled, because fingerprinting is not a risk.
messageHashesToDownloadList := make ( [ ] [ 26 ] byte , 0 )
hostIsMalicious := false
for _ , receivedMessageHash := range retrievedMessageHashesList {
checkIfHostIsOfferingInvalidMessage := func ( ) ( bool , error ) {
messageMetadataExists , _ , messageNetworkType , _ , messageInbox , _ , err := contentMetadata . GetMessageMetadata ( receivedMessageHash )
if ( err != nil ) { return false , err }
if ( messageMetadataExists == false ) {
return false , nil
}
if ( messageNetworkType != networkType ) {
return true , nil
}
if ( len ( requestedInboxesList ) != 0 ) {
isInInboxesList := slices . Contains ( requestedInboxesList , messageInbox )
if ( isInInboxesList == false ) {
return true , nil
}
} else {
inboxIsWithinRange , err := byteRange . CheckIfInboxIsWithinRange ( requestedInboxesRangeStart , requestedInboxesRangeEnd , messageInbox )
if ( err != nil ) { return false , err }
if ( inboxIsWithinRange == false ) {
return true , nil
}
}
return false , nil
}
hostIsOfferingInvalidMessage , err := checkIfHostIsOfferingInvalidMessage ( )
if ( err != nil ) { return false , err }
if ( hostIsOfferingInvalidMessage == true ) {
// Host is malicious, we must still download the message so they do not learn that we have this message downloaded (or did at one point)
hostIsMalicious = true
messageHashesToDownloadList = append ( messageHashesToDownloadList , receivedMessageHash )
continue
}
storedMessageExists , _ , err := badgerDatabase . GetChatMessage ( receivedMessageHash )
if ( err != nil ) { return false , err }
if ( storedMessageExists == true ) {
// We already have the message, skip downloading it
continue
}
// Message does not exist, we must check if we want to download it
// This works differently for Hosting, Moderation, and Viewing
// For instance, if the user is downloading messages for viewing, they will not redownload profiles that they have already imported/deleted
shouldDownload , err := checkIfMessageShouldBeDownloaded ( receivedMessageHash )
if ( err != nil ) { return false , err }
if ( shouldDownload == true ) {
messageHashesToDownloadList = append ( messageHashesToDownloadList , receivedMessageHash )
continue
}
}
if ( len ( messageHashesToDownloadList ) == 0 ) {
// Nothing to download, done with this host.
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
return true , nil
}
// Now we retrieve messages from host
maximumMessagesInResponse := serverResponse . MaximumMessagesInResponse_GetMessages
messageHashesSublistsList , err := helpers . SplitListIntoSublists ( messageHashesToDownloadList , maximumMessagesInResponse )
if ( err != nil ) { return false , err }
numberOfDownloadedMessages := 0
for _ , messageHashesSublist := range messageHashesSublistsList {
downloadSuccessful , messagesList , err := sendRequests . GetMessagesFromHost ( connectionIdentifier , hostIdentityHash , networkType , messageHashesSublist , requestedInboxesRangeStart , requestedInboxesRangeEnd , requestedInboxesList )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Request has failed
// We will stop downloading from this host
break
}
for _ , messageBytes := range messagesList {
messageIsWellFormed , err := chatMessageStorage . AddMessage ( messageBytes )
if ( err != nil ) { return false , err }
if ( messageIsWellFormed == false ) {
return false , errors . New ( "GetMessagesFromHost not verifying messages are well formed." )
}
}
numberOfDownloadedMessages += len ( messagesList )
}
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
if ( numberOfDownloadedMessages != 0 ) {
numberOfMessagesDownloadedString := helpers . ConvertIntToString ( numberOfDownloadedMessages )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfMessagesDownloadedString + " messages from host." )
if ( err != nil ) { return false , err }
}
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadMessagesFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadIdentityReviewsFromHosts (
allowClearnet bool ,
networkType byte ,
identityTypeToRetrieve string ,
rangeToRetrieveStart [ 16 ] byte ,
rangeToRetrieveEnd [ 16 ] byte ,
reviewedIdentityHashesToRetrieveList [ ] [ 16 ] byte ,
reviewersToRetrieveList [ ] [ 16 ] byte ,
checkIfReviewShouldBeDownloaded func ( [ 29 ] byte , [ ] byte ) ( bool , error ) ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadIdentityReviewsFromHosts called with invalid networkType: " + networkTypeString )
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReviewsFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[29]byte][]byte: Map of review info: ReviewHash -> ReviewedHash
// -[16]byte: Expected Identity Start Range of retrieved reviews
// -[16]byte: Expected Identity End range of retrieved reviews
// -[][16]byte: Expected retrieved identity hashes list
// -error
getRetrievedReviewsInfoMap := func ( ) ( bool , [ 21 ] byte , map [ [ 29 ] byte ] [ ] byte , [ 16 ] byte , [ 16 ] byte , [ ] [ 16 ] byte , error ) {
hostIsHostingIdentityType , theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd , err := hostRanges . GetHostedIdentityHashRangeFromHostRawProfileMap ( hostRawProfileMap , identityTypeToRetrieve )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostIsHostingIdentityType == false ) {
// Host is not hosting any profiles/reviews of this identityType. Skip to next host
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
exists , identityTypeIdentitiesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , identityTypeToRetrieve + "IdentitiesQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "IdentitiesQuantity" )
}
identityTypeIdentitiesQuantityInt64 , err := helpers . ConvertStringToInt64 ( identityTypeIdentitiesQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity )
}
getMaximumIdentitiesPerRequest := func ( ) ( int64 , error ) {
maximumReviewsInResponse := serverResponse . MaximumReviewsInResponse_GetIdentityReviewsInfo
if ( len ( reviewersToRetrieveList ) != 0 ) {
// We assume each reviewer has reviewed each identity and all it's profiles
profilesQuantityAttributeName := identityTypeToRetrieve + "ProfilesQuantity"
exists , identityTypeProfilesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , profilesQuantityAttributeName )
if ( err != nil ) { return 0 , err }
if ( exists == false ) {
return 0 , errors . New ( "Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ProfilesQuantity" )
}
identityTypeProfilesQuantityInt64 , err := helpers . ConvertStringToInt64 ( identityTypeProfilesQuantity )
if ( err != nil ) { return 0 , err }
estimatedNumberOfProfilesPerIdentity := identityTypeProfilesQuantityInt64 / identityTypeIdentitiesQuantityInt64
numberOfReviewersToRetrieve := int64 ( len ( reviewersToRetrieveList ) )
estimatedReviewsPerIdentity := numberOfReviewersToRetrieve * ( estimatedNumberOfProfilesPerIdentity + 1 )
maximumIdentitiesPerRequest := int64 ( maximumReviewsInResponse ) / estimatedReviewsPerIdentity
return maximumIdentitiesPerRequest , nil
}
reviewsQuantityAttributeName := identityTypeToRetrieve + "ReviewsQuantity"
exists , identityTypeReviewsQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , reviewsQuantityAttributeName )
if ( err != nil ) { return 0 , err }
if ( exists == false ) {
return 0 , errors . New ( "Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ReviewsQuantity" )
}
identityTypeReviewsQuantityInt64 , err := helpers . ConvertStringToInt64 ( identityTypeReviewsQuantity )
if ( err != nil ) { return 0 , err }
estimatedReviewsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReviewsQuantityInt64
maximumIdentitiesPerRequest := int64 ( maximumReviewsInResponse ) / estimatedReviewsPerIdentity
return maximumIdentitiesPerRequest , nil
}
maximumIdentitiesPerRequest , err := getMaximumIdentitiesPerRequest ( )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
// Our request will either be based on identity hash range or identity hashes list
// We will split up the request into either ranges or identity hashes
if ( len ( reviewedIdentityHashesToRetrieveList ) != 0 ) {
// We will split the request into identity hash lists
hostingAny , identitiesInHostRangeList , err := byteRange . GetAllIdentityHashesInListWithinRange ( theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd , reviewedIdentityHashesToRetrieveList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostingAny == false ) {
// Host is not hosting any of our desired identities. Skip host.
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
// Below step is just being overly cautious.
// The GetIdentityReviewsfromHosts function should request either a list of identities or a range. Not both.
hostingAny , identitiesInRequestRange , err := byteRange . GetAllIdentityHashesInListWithinRange ( rangeToRetrieveStart , rangeToRetrieveEnd , identitiesInHostRangeList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostingAny == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Requested review identity hashes list outside of request range" )
}
sublistsToQueryList , err := helpers . SplitListIntoSublists ( identitiesInRequestRange , int ( maximumIdentitiesPerRequest ) )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
minimumRange , maximumRange := byteRange . GetMinimumMaximumIdentityHashBounds ( )
getDownloadedReviewsInfoMap := func ( ) ( bool , map [ [ 29 ] byte ] [ ] byte , error ) {
retrievedReviewsInfoMap := make ( map [ [ 29 ] byte ] [ ] byte )
for _ , identityHashesSublist := range sublistsToQueryList {
downloadSuccessful , currentRetrievedReviewsInfoMap , err := sendRequests . GetIdentityReviewsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , identityTypeToRetrieve , minimumRange , maximumRange , identityHashesSublist , reviewersToRetrieveList )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop download from host.
err := logger . AddLogEntry ( "Network" , "Failed to download identity reviews info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reviewHash , reviewedHash := range currentRetrievedReviewsInfoMap {
retrievedReviewsInfoMap [ reviewHash ] = reviewedHash
}
}
return true , retrievedReviewsInfoMap , nil
}
downloadSuccessful , retrievedReviewsInfoMap , err := getDownloadedReviewsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedReviewsInfoMap , minimumRange , maximumRange , identitiesInRequestRange , nil
}
// We will retrieve based on identity range, not identities list
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := byteRange . GetIdentityIntersectionRangeFromTwoRanges ( rangeToRetrieveStart , rangeToRetrieveEnd , theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( anyIntersectionFound == false ) {
// Host is not hosting any reviews within the identites range that we are requesting
// Skip this host
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
estimatedNumIdentitiesInIntersectionRange , err := byteRange . GetEstimatedIdentitySubrangeQuantity ( theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd , identityTypeIdentitiesQuantityInt64 , intersectionRangeStart , intersectionRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
subrangesToQueryList , err := byteRange . SplitIdentityRangeIntoEqualSubranges ( intersectionRangeStart , intersectionRangeEnd , estimatedNumIdentitiesInIntersectionRange , maximumIdentitiesPerRequest )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
getDownloadedReviewsInfoMap := func ( ) ( bool , map [ [ 29 ] byte ] [ ] byte , error ) {
retrievedReviewsInfoMap := make ( map [ [ 29 ] byte ] [ ] byte )
for _ , subrangeObject := range subrangesToQueryList {
subrangeStart := subrangeObject . SubrangeStart
subrangeEnd := subrangeObject . SubrangeEnd
emptyList := make ( [ ] [ 16 ] byte , 0 )
downloadSuccessful , currentRetrievedReviewsInfoMap , err := sendRequests . GetIdentityReviewsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , identityTypeToRetrieve , subrangeStart , subrangeEnd , emptyList , reviewersToRetrieveList )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop download.
err := logger . AddLogEntry ( "Network" , "Failed to download identity reviews info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reviewHash , reviewedHash := range currentRetrievedReviewsInfoMap {
retrievedReviewsInfoMap [ reviewHash ] = reviewedHash
}
}
return true , retrievedReviewsInfoMap , nil
}
downloadSuccessful , retrievedReviewsInfoMap , err := getDownloadedReviewsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedReviewsInfoMap , intersectionRangeStart , intersectionRangeEnd , nil , nil
}
hostDownloadSuccessful , connectionIdentifier , retrievedReviewsInfoMap , expectedRangeStart , expectedRangeEnd , expectedReviewedIdentityHashesList , err := getRetrievedReviewsInfoMap ( )
if ( err != nil ) { return false , err }
if ( hostDownloadSuccessful == false ) {
// We did not download any reviews from this host. Skip them.
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
// Now we see if we have any of these reviews, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// This requires checking against content stored in our database
// A host could use this to learn information about what content we are storing
// We still download the reviews to prevent host from knowing that we have the reviewed content downloaded
// Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response
// Only after the full request is finished do we mark the host as malicious
// TODO: We shouldn't have to wait until the end of the request if we are only in Host/Moderator mode
// This is because we don't need to worry about leaking what content we have stored
reviewHashesToDownloadList := make ( [ ] [ 29 ] byte , 0 )
hostIsMalicious := false
for receivedReviewHash , receivedReviewedHash := range retrievedReviewsInfoMap {
storedReviewExists , storedReviewBytes , err := badgerDatabase . GetReview ( receivedReviewHash )
if ( err != nil ) { return false , err }
checkIfHostIsOfferingInvalidReview := func ( ) ( bool , error ) {
if ( storedReviewExists == true ) {
ableToRead , storedReviewHash , _ , reviewNetworkType , storedReviewReviewerIdentity , _ , storedReviewType , storedReviewReviewedHash , _ , _ , err := readReviews . ReadReviewAndHash ( false , storedReviewBytes )
if ( err != nil ) { return false , err }
if ( ableToRead == false ) {
return false , errors . New ( "Database corrupt: Contains invalid review." )
}
if ( storedReviewHash != receivedReviewHash ) {
return false , errors . New ( "Database corrupt: Contains review with invalid entry key hash" )
}
if ( reviewNetworkType != networkType ) {
return true , nil
}
if ( len ( reviewersToRetrieveList ) != 0 ) {
isInList := slices . Contains ( reviewersToRetrieveList , storedReviewReviewerIdentity )
if ( isInList == false ) {
return true , nil
}
}
areEqual := bytes . Equal ( storedReviewReviewedHash , receivedReviewedHash )
if ( areEqual == false ) {
return true , nil
}
if ( storedReviewType != "Identity" && storedReviewType != "Profile" && storedReviewType != "Attribute" ) {
return true , nil
}
}
// Now we check to see if we are aware of the reviewed content's metadata, and if it matches our request parameters
receivedReviewedType , err := helpers . GetReviewedTypeFromReviewedHash ( receivedReviewedHash )
if ( err != nil ) { return false , err }
if ( receivedReviewedType == "Profile" ) {
// We must verify that the profile's identity hash is expected
// We don't have to do this for identities, because we already checked that in the GetIdentityReviewsInfo function
if ( len ( receivedReviewedHash ) != 28 ) {
receivedReviewedHashHex := encoding . EncodeBytesToHexString ( receivedReviewedHash )
return false , errors . New ( "GetReviewedTypeFromReviewedHash returning Profile for different length reviewedHash: " + receivedReviewedHashHex )
}
receivedReviewedProfileHash := [ 28 ] byte ( receivedReviewedHash )
metadataIsKnown , _ , profileNetworkType , reviewedProfileAuthor , _ , profileIsDisabled , _ , _ , err := contentMetadata . GetProfileMetadata ( receivedReviewedProfileHash )
if ( err != nil ) { return false , err }
if ( metadataIsKnown == false ) {
return false , nil
}
if ( profileIsDisabled == true ) {
// Disabled profiles cannot be reviewed. Host must be malicious.
return true , nil
}
if ( profileNetworkType != networkType ) {
// Review is reviewing profile from different networkType
// Host should have checked this already. Host must be malicious.
return true , nil
}
if ( expectedReviewedIdentityHashesList != nil ) {
identityIsInList := slices . Contains ( expectedReviewedIdentityHashesList , reviewedProfileAuthor )
if ( identityIsInList == false ) {
return true , nil
}
} else {
identityIsWithinRange , err := byteRange . CheckIfIdentityHashIsWithinRange ( expectedRangeStart , expectedRangeEnd , reviewedProfileAuthor )
if ( err != nil ) { return false , err }
if ( identityIsWithinRange == false ) {
return true , nil
}
}
}
if ( receivedReviewedType == "Attribute" ) {
if ( len ( receivedReviewedHash ) != 27 ) {
receivedReviewedHashHex := encoding . EncodeBytesToHexString ( receivedReviewedHash )
return false , errors . New ( "GetReviewedTypeFromReviewedHash returning Attribute for different length reviewedHash: " + receivedReviewedHashHex )
}
receivedReviewedAttributeHash := [ 27 ] byte ( receivedReviewedHash )
metadataExists , _ , attributeAuthorIdentityHash , attributeNetworkType , _ , err := profileStorage . GetProfileAttributeMetadata ( receivedReviewedAttributeHash )
if ( err != nil ) { return false , err }
if ( metadataExists == true ) {
if ( expectedReviewedIdentityHashesList != nil ) {
identityIsInList := slices . Contains ( expectedReviewedIdentityHashesList , attributeAuthorIdentityHash )
if ( identityIsInList == false ) {
return true , nil
}
} else {
identityIsWithinRange , err := byteRange . CheckIfIdentityHashIsWithinRange ( expectedRangeStart , expectedRangeEnd , attributeAuthorIdentityHash )
if ( err != nil ) { return false , err }
if ( identityIsWithinRange == false ) {
return true , nil
}
}
if ( attributeNetworkType != networkType ) {
// Review is reviewing attribute from different networkType
// Host should have checked this already. Host must be malicious.
return true , nil
}
return false , nil
}
}
// Review is not invalid, as far as we know
return false , nil
}
hostIsOfferingInvalidReview , err := checkIfHostIsOfferingInvalidReview ( )
if ( err != nil ) { return false , err }
if ( hostIsOfferingInvalidReview == true ) {
// Host is malicious, we must still download the review
hostIsMalicious = true
reviewHashesToDownloadList = append ( reviewHashesToDownloadList , receivedReviewHash )
continue
}
if ( storedReviewExists == true ) {
// We already have the review, skip downloading it
continue
}
// Review does not exist, we must check if we want to download it
shouldDownload , err := checkIfReviewShouldBeDownloaded ( receivedReviewHash , receivedReviewedHash )
if ( err != nil ) { return false , err }
if ( shouldDownload == true ) {
reviewHashesToDownloadList = append ( reviewHashesToDownloadList , receivedReviewHash )
continue
}
}
if ( len ( reviewHashesToDownloadList ) == 0 ) {
// Nothing to download, we are done with this host.
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
return true , nil
}
// Now we download reviews
maximumReviewsInResponse := serverResponse . MaximumReviewsInResponse_GetReviews
reviewHashesSublistsList , err := helpers . SplitListIntoSublists ( reviewHashesToDownloadList , maximumReviewsInResponse )
if ( err != nil ) { return false , err }
numberOfDownloadedReviews := 0
for _ , reviewHashesSublist := range reviewHashesSublistsList {
downloadSuccessful , reviewsList , err := sendRequests . GetReviewsFromHost ( connectionIdentifier , hostIdentityHash , networkType , reviewHashesSublist , retrievedReviewsInfoMap , reviewersToRetrieveList )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Request has failed
// We will stop downloading from this host
break
}
if ( hostIsMalicious == true ) {
// We don't add reviews from malicious hosts
continue
}
for _ , reviewBytes := range reviewsList {
reviewIsWellFormed , err := reviewStorage . AddReview ( reviewBytes )
if ( err != nil ) { return false , err }
if ( reviewIsWellFormed == false ) {
return false , errors . New ( "GetReviewsFromHost not verifying reviews are well formed." )
}
}
numberOfDownloadedReviews += len ( reviewsList )
}
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
if ( numberOfDownloadedReviews != 0 ) {
numberOfReviewsDownloadedString := helpers . ConvertIntToString ( numberOfDownloadedReviews )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfReviewsDownloadedString + " identity reviews from host." )
if ( err != nil ) { return false , err }
}
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadReviewsFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadMessageReviewsFromHosts (
allowClearnet bool ,
networkType byte ,
rangeToRetrieveStart [ 10 ] byte ,
rangeToRetrieveEnd [ 10 ] byte ,
reviewedMessageHashesToRetrieveList [ ] [ 26 ] byte ,
reviewersToRetrieveList [ ] [ 16 ] byte ,
reviewedMessageInboxesMap map [ [ 26 ] byte ] [ 10 ] byte , // Message hash -> Message inbox
checkIfReviewShouldBeDownloaded func ( [ 29 ] byte , [ 26 ] byte ) ( bool , error ) ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadMessageReviewsFromHosts called with invalid networkType: " + networkTypeString )
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReviewsFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[29]byte][26]byte: Map of review info: ReviewHash -> ReviewedHash
// -[10]byte: Expected Inbox Start Range of retrieved reviews
// -[10]byte: Expected Inbox End range of retrieved reviews
// -error
getRetrievedReviewsInfoMap := func ( ) ( bool , [ 21 ] byte , map [ [ 29 ] byte ] [ 26 ] byte , [ 10 ] byte , [ 10 ] byte , error ) {
hostIsHostingMessages , theirInboxRangeStart , theirInboxRangeEnd , err := hostRanges . GetHostedMessageInboxesRangeFromHostRawProfileMap ( hostRawProfileMap )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( hostIsHostingMessages == false ) {
// Host is not hosting messages. Nothing to download.
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
exists , hostedMessagesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "MessagesQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile missing MessagesQuantity" )
}
hostedMessagesQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedMessagesQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity )
}
exists , hostedMessageReviewsQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "MessageReviewsQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile missing MessageReviewsQuantity" )
}
hostedMessageReviewsQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedMessageReviewsQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile with invalid MessageReviewsQuantity: " + hostedMessageReviewsQuantity )
}
getMaximumMessageHashesPerRequest := func ( ) int {
maximumReviewsInResponse := serverResponse . MaximumReviewsInResponse_GetMessageReviewsInfo
if ( len ( reviewersToRetrieveList ) != 0 ) {
// We assume each reviewer has 1 review per message
numberOfReviewersToRetrieve := len ( reviewersToRetrieveList )
maximumMessageHashesPerRequest := maximumReviewsInResponse / numberOfReviewersToRetrieve
return maximumMessageHashesPerRequest
}
// We get the estimated number of reviews per message
estimatedReviewsPerMessage := hostedMessageReviewsQuantityInt64 / hostedMessagesQuantityInt64
maximumMessageHashesPerRequest := int64 ( maximumReviewsInResponse ) / estimatedReviewsPerMessage
return int ( maximumMessageHashesPerRequest )
}
maximumMessageHashesPerRequest := getMaximumMessageHashesPerRequest ( )
// We will retrieve reviews from either the reviewed hashes list, or request range
if ( len ( reviewedMessageHashesToRetrieveList ) != 0 ) {
// We will retrieve reviewed hashes list
messageHashesInTheirRangeList := make ( [ ] [ 26 ] byte , 0 )
for _ , messageHash := range reviewedMessageHashesToRetrieveList {
messageInbox , exists := reviewedMessageInboxesMap [ messageHash ]
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "reviewedMessageInboxesMap missing message hash." )
}
inboxIsWithinTheirRange , err := byteRange . CheckIfInboxIsWithinRange ( theirInboxRangeStart , theirInboxRangeEnd , messageInbox )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( inboxIsWithinTheirRange == true ) {
messageHashesInTheirRangeList = append ( messageHashesInTheirRangeList , messageHash )
}
}
sublistsToQueryList , err := helpers . SplitListIntoSublists ( messageHashesInTheirRangeList , maximumMessageHashesPerRequest )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
minimumRange , maximumRange := byteRange . GetMinimumMaximumInboxBounds ( )
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
//Outputs:
// -bool: Download successful
// -map[[29]byte][26]byte: Reviews info map (Review hash -> Reviewed message hash)
// -error
getDownloadedReviewsInfoMap := func ( ) ( bool , map [ [ 29 ] byte ] [ 26 ] byte , error ) {
retrievedReviewsInfoMap := make ( map [ [ 29 ] byte ] [ 26 ] byte )
for _ , messageHashesSublist := range sublistsToQueryList {
downloadSuccessful , currentRetrievedReviewsInfoMap , err := sendRequests . GetMessageReviewsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , minimumRange , maximumRange , messageHashesSublist , reviewersToRetrieveList )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop retrieving from this host
err := logger . AddLogEntry ( "Network" , "Failed to download message reviews info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reviewHash , reviewedHash := range currentRetrievedReviewsInfoMap {
retrievedReviewsInfoMap [ reviewHash ] = reviewedHash
}
}
return true , retrievedReviewsInfoMap , nil
}
downloadSuccessful , retrievedReviewsInfoMap , err := getDownloadedReviewsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errB }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
return true , connectionIdentifier , retrievedReviewsInfoMap , minimumRange , maximumRange , nil
}
// We will retrieve reviews based on message inbox range
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := byteRange . GetInboxIntersectionRangeFromTwoRanges ( rangeToRetrieveStart , rangeToRetrieveEnd , theirInboxRangeStart , theirInboxRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( anyIntersectionFound == false ) {
// Host is not hosting any reviews within the inboxes range that we are requesting
// Skip this host
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
estimatedNumItemsInIntersectionRange , err := byteRange . GetEstimatedInboxSubrangeQuantity ( theirInboxRangeStart , theirInboxRangeEnd , hostedMessagesQuantityInt64 , intersectionRangeStart , intersectionRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
subrangesToQueryList , err := byteRange . SplitInboxRangeIntoEqualSubranges ( intersectionRangeStart , intersectionRangeEnd , estimatedNumItemsInIntersectionRange , int64 ( maximumMessageHashesPerRequest ) )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
//Outputs:
// -bool: Download successful
// -map[[29]byte][26]byte: Downloaded reviews info map (Review Hash -> Reviewed Message Hash)
// -error
getDownloadedReviewsInfoMap := func ( ) ( bool , map [ [ 29 ] byte ] [ 26 ] byte , error ) {
retrievedReviewsInfoMap := make ( map [ [ 29 ] byte ] [ 26 ] byte )
for _ , subrangeObject := range subrangesToQueryList {
subrangeStart := subrangeObject . SubrangeStart
subrangeEnd := subrangeObject . SubrangeEnd
emptyList := make ( [ ] [ 26 ] byte , 0 )
downloadSuccessful , currentRetrievedReviewsInfoMap , err := sendRequests . GetMessageReviewsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , subrangeStart , subrangeEnd , emptyList , reviewersToRetrieveList )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, we are done with this host
err := logger . AddLogEntry ( "Network" , "Failed to download message reviews info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reviewHash , reviewedHash := range currentRetrievedReviewsInfoMap {
retrievedReviewsInfoMap [ reviewHash ] = reviewedHash
}
}
return true , retrievedReviewsInfoMap , nil
}
downloadSuccessful , retrievedReviewsInfoMap , err := getDownloadedReviewsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errB }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
return true , connectionIdentifier , retrievedReviewsInfoMap , intersectionRangeStart , intersectionRangeEnd , nil
}
hostDownloadSuccessful , connectionIdentifier , retrievedReviewsInfoMap , expectedRangeStart , expectedRangeEnd , err := getRetrievedReviewsInfoMap ( )
if ( err != nil ) { return false , err }
if ( hostDownloadSuccessful == false ) {
// We did not download any reviews from this host. Skip them.
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
// Now we see if we have any of these reviews, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// This requires checking against content stored in our database
// A host could use this to learn information about what messages we are storing
// We still download the reviews to prevent host from knowing that we have the reviewed messages downloaded
// Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response
// Only after the full request is finished do we mark the host as malicious
// TODO: We shouldn't have to wait until the end of the request if we are only in Host mode
// This is because if we we don't need to worry about leaking what messages we have stored
reviewHashesToDownloadList := make ( [ ] [ 29 ] byte , 0 )
hostIsMalicious := false
for receivedReviewHash , receivedReviewedHash := range retrievedReviewsInfoMap {
storedReviewExists , storedReviewBytes , err := badgerDatabase . GetReview ( receivedReviewHash )
if ( err != nil ) { return false , err }
checkIfHostIsOfferingInvalidReview := func ( ) ( bool , error ) {
if ( storedReviewExists == true ) {
ableToRead , storedReviewHash , _ , reviewNetworkType , storedReviewReviewerIdentity , _ , storedReviewType , storedReviewReviewedHash , _ , _ , err := readReviews . ReadReviewAndHash ( false , storedReviewBytes )
if ( err != nil ) { return false , err }
if ( ableToRead == false ) {
return false , errors . New ( "Database corrupt: Contains invalid review." )
}
if ( storedReviewHash != receivedReviewHash ) {
return false , errors . New ( "Database corrupt: Contains review with invalid entry key hash" )
}
if ( reviewNetworkType != networkType ) {
return true , nil
}
if ( len ( reviewersToRetrieveList ) != 0 ) {
isInList := slices . Contains ( reviewersToRetrieveList , storedReviewReviewerIdentity )
if ( isInList == false ) {
return true , nil
}
}
areEqual := bytes . Equal ( storedReviewReviewedHash , receivedReviewedHash [ : ] )
if ( areEqual == false ) {
return true , nil
}
if ( storedReviewType != "Message" ) {
return true , nil
}
}
// Now we check to see if we are aware of the reviewed message's metadata, and if it matches our request parameters
metadataIsKnown , _ , messageNetworkType , _ , reviewedMessageInbox , _ , err := contentMetadata . GetMessageMetadata ( receivedReviewedHash )
if ( err != nil ) { return false , err }
if ( metadataIsKnown == false ) {
return false , nil
}
if ( messageNetworkType != networkType ) {
// The review is reviewing a message from a different networkType
// The host should have checked for this before serving us the review
// Thus, the host must be malicious
return true , nil
}
messageInboxIsWithinRange , err := byteRange . CheckIfInboxIsWithinRange ( expectedRangeStart , expectedRangeEnd , reviewedMessageInbox )
if ( err != nil ) { return false , err }
if ( messageInboxIsWithinRange == false ) {
return false , nil
}
return false , nil
}
hostIsOfferingInvalidReview , err := checkIfHostIsOfferingInvalidReview ( )
if ( err != nil ) { return false , err }
if ( hostIsOfferingInvalidReview == true ) {
// Host is malicious, we must still download the review
hostIsMalicious = true
reviewHashesToDownloadList = append ( reviewHashesToDownloadList , receivedReviewHash )
continue
}
if ( storedReviewExists == true ) {
// We already have the review, skip downloading it
continue
}
// Review does not exist, we must check if we want to download it
shouldDownload , err := checkIfReviewShouldBeDownloaded ( receivedReviewHash , receivedReviewedHash )
if ( err != nil ) { return false , err }
if ( shouldDownload == true ) {
reviewHashesToDownloadList = append ( reviewHashesToDownloadList , receivedReviewHash )
continue
}
}
if ( len ( reviewHashesToDownloadList ) == 0 ) {
// Nothing to download, done with this host.
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
return true , nil
}
// Now we download reviews
// We convert the [26]byte values to []byte values to pass to GetReviewsFromHost
reviewsInfoMapForRequest := make ( map [ [ 29 ] byte ] [ ] byte )
for reviewHash , reviewedMessageHash := range retrievedReviewsInfoMap {
reviewsInfoMapForRequest [ reviewHash ] = reviewedMessageHash [ : ]
}
maximumReviewsInResponse := serverResponse . MaximumReviewsInResponse_GetReviews
reviewHashesSublistsList , err := helpers . SplitListIntoSublists ( reviewHashesToDownloadList , maximumReviewsInResponse )
if ( err != nil ) { return false , err }
numberOfDownloadedReviews := 0
for _ , reviewHashesSublist := range reviewHashesSublistsList {
downloadSuccessful , reviewsList , err := sendRequests . GetReviewsFromHost ( connectionIdentifier , hostIdentityHash , networkType , reviewHashesSublist , reviewsInfoMapForRequest , reviewersToRetrieveList )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Request has failed
// We will stop downloading from this host
break
}
if ( hostIsMalicious == true ) {
// We don't add reviews from malicious hosts
continue
}
for _ , reviewBytes := range reviewsList {
reviewIsWellFormed , err := reviewStorage . AddReview ( reviewBytes )
if ( err != nil ) { return false , err }
if ( reviewIsWellFormed == false ) {
return false , errors . New ( "GetReviewsFromHost not verifying reviews are well formed." )
}
}
numberOfDownloadedReviews += len ( reviewsList )
}
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
if ( numberOfDownloadedReviews != 0 ) {
numberOfReviewsDownloadedString := helpers . ConvertIntToString ( numberOfDownloadedReviews )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfReviewsDownloadedString + " reviews from host." )
if ( err != nil ) { return false , err }
}
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadReviewsFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadIdentityReportsFromHosts (
allowClearnet bool ,
networkType byte ,
identityTypeToRetrieve string ,
rangeToRetrieveStart [ 16 ] byte ,
rangeToRetrieveEnd [ 16 ] byte ,
reportedIdentityHashesToRetrieveList [ ] [ 16 ] byte ,
checkIfReportShouldBeDownloaded func ( [ 30 ] byte , [ ] byte ) ( bool , error ) ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadIdentityReportsFromHosts called with invalid networkType: " + networkTypeString )
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReportsFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[30]byte][]byte: Map of report info: ReportHash -> ReportedHash
// -[16]byte: Expected Identity Start Range of retrieved reports
// -[16]byte: Expected Identity End range of retrieved reports
// -[][16]byte: Expected retrieved identity hashes list
// -error
getRetrievedReportsInfoMap := func ( ) ( bool , [ 21 ] byte , map [ [ 30 ] byte ] [ ] byte , [ 16 ] byte , [ 16 ] byte , [ ] [ 16 ] byte , error ) {
hostIsHostingIdentityType , theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd , err := hostRanges . GetHostedIdentityHashRangeFromHostRawProfileMap ( hostRawProfileMap , identityTypeToRetrieve )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostIsHostingIdentityType == false ) {
// Host is not hosting any profiles/reports of this identityType. Skip to next host
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
identitiesQuantityAttributeName := identityTypeToRetrieve + "IdentitiesQuantity"
exists , identityTypeIdentitiesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , identitiesQuantityAttributeName )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile missing " + identitiesQuantityAttributeName )
}
identityTypeIdentitiesQuantityInt64 , err := helpers . ConvertStringToInt64 ( identityTypeIdentitiesQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Database corrupt: Contains Host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity )
}
reportsQuantityAttributeName := identityTypeToRetrieve + "ReportsQuantity"
exists , identityTypeReportsQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , reportsQuantityAttributeName )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile missing " + reportsQuantityAttributeName )
}
identityTypeReportsQuantityInt64 , err := helpers . ConvertStringToInt64 ( identityTypeReportsQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Database corrupt: Contains host profile with invalid " + reportsQuantityAttributeName + ": " + identityTypeReportsQuantity )
}
//TODO: Deal with divide by 0 case here and on all other functions
estimatedReportsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReportsQuantityInt64
maximumReportsInResponse := serverResponse . MaximumReportsInResponse_GetIdentityReportsInfo
maximumIdentitiesPerRequest := int64 ( maximumReportsInResponse ) / estimatedReportsPerIdentity
// Our request will either be based on identity hash range or identity hashes list
// We will split up the request into either ranges or identity hash lists
if ( len ( reportedIdentityHashesToRetrieveList ) != 0 ) {
// We will split the request into identity hash lists
hostingAny , identitiesInHostRangeList , err := byteRange . GetAllIdentityHashesInListWithinRange ( theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd , reportedIdentityHashesToRetrieveList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostingAny == false ) {
// Host is not hosting any of our desired identities. Skip host.
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
// Below step is just being overly cautious.
// The GetMessageReportsfromHosts function should request either a list of identities or a range. Not both.
hostingAny , identitiesInRequestRange , err := byteRange . GetAllIdentityHashesInListWithinRange ( rangeToRetrieveStart , rangeToRetrieveEnd , identitiesInHostRangeList )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostingAny == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errors . New ( "Requested report identity hashes list outside of request range" )
}
sublistsToQueryList , err := helpers . SplitListIntoSublists ( identitiesInRequestRange , int ( maximumIdentitiesPerRequest ) )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
minimumRange , maximumRange := byteRange . GetMinimumMaximumIdentityHashBounds ( )
//Outputs:
// -bool: Download successful
// -map[[30]byte][]byte: Downloaded reports info map (Report Hash -> Reported Hash)
// -error
getDownloadedReportsInfoMap := func ( ) ( bool , map [ [ 30 ] byte ] [ ] byte , error ) {
retrievedReportsInfoMap := make ( map [ [ 30 ] byte ] [ ] byte )
for _ , identityHashesSublist := range sublistsToQueryList {
downloadSuccessful , currentRetrievedReportsInfoMap , err := sendRequests . GetIdentityReportsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , identityTypeToRetrieve , minimumRange , maximumRange , identityHashesSublist )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop downloading from this host
err := logger . AddLogEntry ( "Network" , "Failed to download identity reports info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reportHash , reportedHash := range currentRetrievedReportsInfoMap {
retrievedReportsInfoMap [ reportHash ] = reportedHash
}
}
return true , retrievedReportsInfoMap , nil
}
downloadSuccessful , retrievedReportsInfoMap , err := getDownloadedReportsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedReportsInfoMap , minimumRange , maximumRange , identitiesInRequestRange , nil
}
// We will retrieve based on identity range, not identities list
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := byteRange . GetIdentityIntersectionRangeFromTwoRanges ( rangeToRetrieveStart , rangeToRetrieveEnd , theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( anyIntersectionFound == false ) {
// Host is not hosting any reports within the identites range that we are requesting
// Skip this host
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
estimatedNumIdentitiesInIntersectionRange , err := byteRange . GetEstimatedIdentitySubrangeQuantity ( theirHostedIdentitiesRangeStart , theirHostedIdentitiesRangeEnd , identityTypeIdentitiesQuantityInt64 , intersectionRangeStart , intersectionRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
subrangesToQueryList , err := byteRange . SplitIdentityRangeIntoEqualSubranges ( intersectionRangeStart , intersectionRangeEnd , estimatedNumIdentitiesInIntersectionRange , maximumIdentitiesPerRequest )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
getDownloadedReportsInfoMap := func ( ) ( bool , map [ [ 30 ] byte ] [ ] byte , error ) {
retrievedReportsInfoMap := make ( map [ [ 30 ] byte ] [ ] byte )
for _ , subrangeObject := range subrangesToQueryList {
subrangeStart := subrangeObject . SubrangeStart
subrangeEnd := subrangeObject . SubrangeEnd
emptyList := make ( [ ] [ 16 ] byte , 0 )
downloadSuccessful , currentRetrievedReportsInfoMap , err := sendRequests . GetIdentityReportsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , identityTypeToRetrieve , subrangeStart , subrangeEnd , emptyList )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, close connection to host.
err := logger . AddLogEntry ( "Network" , "Failed to download identity reports info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reportHash , reportedHash := range currentRetrievedReportsInfoMap {
retrievedReportsInfoMap [ reportHash ] = reportedHash
}
}
return true , retrievedReportsInfoMap , nil
}
downloadSuccessful , retrievedReportsInfoMap , err := getDownloadedReportsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , errB }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , err }
return false , [ 21 ] byte { } , nil , [ 16 ] byte { } , [ 16 ] byte { } , nil , nil
}
return true , connectionIdentifier , retrievedReportsInfoMap , intersectionRangeStart , intersectionRangeEnd , nil , nil
}
hostDownloadSuccessful , connectionIdentifier , retrievedReportsInfoMap , expectedRangeStart , expectedRangeEnd , expectedReportedIdentityHashesList , err := getRetrievedReportsInfoMap ( )
if ( err != nil ) { return false , err }
if ( hostDownloadSuccessful == false ) {
// We did not download any reports from this host. Skip them.
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
// Now we see if we have any of these reports, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// We still download the reports to prevent host from knowing that we have those reports downloaded
// We then mark the host as malicious
reportHashesToDownloadList := make ( [ ] [ 30 ] byte , 0 )
hostIsMalicious := false
for receivedReportHash , receivedReportedHash := range retrievedReportsInfoMap {
//TODO: Ability to disable this step if only 1 of Host/Moderator mode is enabled
storedReportExists , storedReportBytes , err := badgerDatabase . GetReport ( receivedReportHash )
if ( err != nil ) { return false , err }
checkIfHostIsOfferingInvalidReport := func ( ) ( bool , error ) {
if ( storedReportExists == true ) {
ableToRead , storedReportHash , _ , reportNetworkType , _ , storedReportType , storedReportReportedHash , _ , err := readReports . ReadReportAndHash ( false , storedReportBytes )
if ( err != nil ) { return false , err }
if ( ableToRead == false ) {
return false , errors . New ( "Database corrupt: Contains invalid report" )
}
if ( storedReportHash != receivedReportHash ) {
return false , errors . New ( "Database corrupt: Contains report with invalid entry key hash" )
}
if ( reportNetworkType != networkType ) {
return true , nil
}
areEqual := bytes . Equal ( storedReportReportedHash , receivedReportedHash )
if ( areEqual == false ) {
return true , nil
}
if ( storedReportType != "Identity" && storedReportType != "Profile" && storedReportType != "Attribute" ) {
return true , nil
}
}
// Now we see if we have the reviewed profile metadata
reportedType , err := helpers . GetReportedTypeFromReportedHash ( receivedReportedHash )
if ( err != nil ) { return false , err }
if ( reportedType == "Profile" ) {
if ( len ( receivedReportedHash ) != 28 ) {
reportedHashHex := encoding . EncodeBytesToHexString ( receivedReportedHash )
return false , errors . New ( "GetReportedTypeFromReportedHash returning Profile for different length reportedHash: " + reportedHashHex )
}
receivedReportedProfileHash := [ 28 ] byte ( receivedReportedHash )
metadataExists , _ , profileNetworkType , profileAuthor , _ , profileIsDisabled , _ , _ , err := contentMetadata . GetProfileMetadata ( receivedReportedProfileHash )
if ( err != nil ) { return false , err }
if ( metadataExists == false ) {
return false , nil
}
if ( profileNetworkType != networkType ) {
// Reported profile belongs to a different networkType than the report
// The host should have checked for this before serving this report
// Host must be malicious
return true , nil
}
if ( profileIsDisabled == true ) {
// Disabled profiles cannot be reported
// Host must be malicious
return true , nil
}
if ( expectedReportedIdentityHashesList != nil ) {
identityIsInList := slices . Contains ( expectedReportedIdentityHashesList , profileAuthor )
if ( identityIsInList == false ) {
return true , nil
}
} else {
identityIsWithinRange , err := byteRange . CheckIfIdentityHashIsWithinRange ( expectedRangeStart , expectedRangeEnd , profileAuthor )
if ( err != nil ) { return false , err }
if ( identityIsWithinRange == false ) {
return true , nil
}
}
} else if ( reportedType == "Attribute" ) {
if ( len ( receivedReportedHash ) != 27 ) {
reportedHashHex := encoding . EncodeBytesToHexString ( receivedReportedHash )
return false , errors . New ( "GetReportedTypeFromReportedHash returning Attribute for different length reportedHash: " + reportedHashHex )
}
receivedReportedAttributeHash := [ 27 ] byte ( receivedReportedHash )
metadataExists , _ , attributeAuthorIdentityHash , attributeNetworkType , _ , err := profileStorage . GetProfileAttributeMetadata ( receivedReportedAttributeHash )
if ( err != nil ) { return false , err }
if ( metadataExists == true ) {
if ( expectedReportedIdentityHashesList != nil ) {
identityIsInList := slices . Contains ( expectedReportedIdentityHashesList , attributeAuthorIdentityHash )
if ( identityIsInList == false ) {
return true , nil
}
} else {
identityIsWithinRange , err := byteRange . CheckIfIdentityHashIsWithinRange ( expectedRangeStart , expectedRangeEnd , attributeAuthorIdentityHash )
if ( err != nil ) { return false , err }
if ( identityIsWithinRange == false ) {
return true , nil
}
}
if ( attributeNetworkType != networkType ) {
// Report is reporting attribute from different networkType
// Host should have checked this already. Host must be malicious.
return true , nil
}
return false , nil
}
}
// Host is not offering invalid report (as far as we know)
return false , nil
}
hostIsOfferingInvalidReport , err := checkIfHostIsOfferingInvalidReport ( )
if ( err != nil ) { return false , err }
if ( hostIsOfferingInvalidReport == true ) {
// Host is malicious, we must still download the report
hostIsMalicious = true
reportHashesToDownloadList = append ( reportHashesToDownloadList , receivedReportHash )
continue
}
if ( storedReportExists == true ) {
// We already have the report, skip downloading it
continue
}
// Report does not exist, we must check if we want to download it
// This works differently for Hosting and Moderation
shouldDownload , err := checkIfReportShouldBeDownloaded ( receivedReportHash , receivedReportedHash )
if ( err != nil ) { return false , err }
if ( shouldDownload == true ) {
reportHashesToDownloadList = append ( reportHashesToDownloadList , receivedReportHash )
continue
}
}
if ( len ( reportHashesToDownloadList ) == 0 ) {
// Nothing to download, done with this host.
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
return true , nil
}
// Now we download reports
maximumReportsInResponse := serverResponse . MaximumReportsInResponse_GetReports
reportHashesSublistsList , err := helpers . SplitListIntoSublists ( reportHashesToDownloadList , maximumReportsInResponse )
if ( err != nil ) { return false , err }
numberOfDownloadedReports := 0
for _ , reportHashesSublist := range reportHashesSublistsList {
downloadSuccessful , reportsList , err := sendRequests . GetReportsFromHost ( connectionIdentifier , hostIdentityHash , networkType , reportHashesSublist , retrievedReportsInfoMap )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Request has failed
// We will stop downloading from this host
break
}
for _ , reportBytes := range reportsList {
reportIsWellFormed , err := reportStorage . AddReport ( reportBytes )
if ( err != nil ) { return false , err }
if ( reportIsWellFormed == false ) {
return false , errors . New ( "GetReportsFromHost not verifying reports are well formed." )
}
}
numberOfDownloadedReports += len ( reportsList )
}
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
if ( numberOfDownloadedReports != 0 ) {
numberOfReportsDownloadedString := helpers . ConvertIntToString ( numberOfDownloadedReports )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfReportsDownloadedString + " identity reports from host." )
if ( err != nil ) { return false , err }
}
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadReportsFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadMessageReportsFromHosts (
allowClearnet bool ,
networkType byte ,
rangeToRetrieveStart [ 10 ] byte ,
rangeToRetrieveEnd [ 10 ] byte ,
reportedMessageHashesToRetrieveList [ ] [ 26 ] byte ,
reportedMessagesInboxMap map [ [ 26 ] byte ] [ 10 ] byte , // Message hash -> Message inbox
checkIfReportShouldBeDownloaded func ( [ 30 ] byte , [ 26 ] byte ) ( bool , error ) ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadMessageReportsFromHosts called with invalid networkType: " + networkTypeString )
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReportsFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[30]byte][26]byte: Map of report info: ReportHash-> Reported message Hash
// -[10]byte: Expected Inbox Start Range of retrieved reports
// -[10]byte: Expected Inbox End range of retrieved reports
// -error
getRetrievedReportsInfoMap := func ( ) ( bool , [ 21 ] byte , map [ [ 30 ] byte ] [ 26 ] byte , [ 10 ] byte , [ 10 ] byte , error ) {
hostIsHostingMessages , theirInboxHostRangeStart , theirInboxHostRangeEnd , err := hostRanges . GetHostedMessageInboxesRangeFromHostRawProfileMap ( hostRawProfileMap )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( hostIsHostingMessages == false ) {
// Host is not hosting messages. Nothing to download.
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
exists , hostedMessagesQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "MessagesQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile missing MessagesQuantity" )
}
hostedMessagesQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedMessagesQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity )
}
exists , hostedMessageReportsQuantity , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "MessageReportsQuantity" )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Database corrupt: Contains host profile missing MessageReportsQuantity" )
}
hostedMessageReportsQuantityInt64 , err := helpers . ConvertStringToInt64 ( hostedMessageReportsQuantity )
if ( err != nil ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "Databse corrupt: Contains host profile with invalid MessageReportsQuantity: " + hostedMessageReportsQuantity )
}
if ( hostedMessageReportsQuantityInt64 == 0 ) {
// Host is not hosting any message reports. Nothing to download.
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
maximumReportsInResponse := serverResponse . MaximumReportsInResponse_GetMessageReportsInfo
estimatedReportsPerMessage := hostedMessageReportsQuantityInt64 / hostedMessagesQuantityInt64
maximumMessageHashesPerRequest := int64 ( maximumReportsInResponse ) / estimatedReportsPerMessage
// We will retrieve reports from either the reported hashes list, or request range
if ( len ( reportedMessageHashesToRetrieveList ) != 0 ) {
// We will retrieve reported message hashes list
messageHashesInTheirRangeList := make ( [ ] [ 26 ] byte , 0 )
for _ , messageHash := range reportedMessageHashesToRetrieveList {
messageInbox , exists := reportedMessagesInboxMap [ messageHash ]
if ( exists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errors . New ( "ReportedMessagesInboxMap missing message hash." )
}
inboxIsWithinTheirRange , err := byteRange . CheckIfInboxIsWithinRange ( theirInboxHostRangeStart , theirInboxHostRangeEnd , messageInbox )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( inboxIsWithinTheirRange == true ) {
messageHashesInTheirRangeList = append ( messageHashesInTheirRangeList , messageHash )
}
}
sublistsToQueryList , err := helpers . SplitListIntoSublists ( messageHashesInTheirRangeList , int ( maximumMessageHashesPerRequest ) )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
minimumRange , maximumRange := byteRange . GetMinimumMaximumInboxBounds ( )
//Outputs:
// -bool: Download successful
// -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported Message Hash)
// -error
getDownloadedReportsInfoMap := func ( ) ( bool , map [ [ 30 ] byte ] [ 26 ] byte , error ) {
retrievedReportsInfoMap := make ( map [ [ 30 ] byte ] [ 26 ] byte )
for _ , messageHashesSublist := range sublistsToQueryList {
downloadSuccessful , currentRetrievedReportsInfoMap , err := sendRequests . GetMessageReportsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , minimumRange , maximumRange , messageHashesSublist )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, end connection to host
err := logger . AddLogEntry ( "Network" , "Failed to download message reports info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reportHash , reportedHash := range currentRetrievedReportsInfoMap {
retrievedReportsInfoMap [ reportHash ] = reportedHash
}
}
return true , retrievedReportsInfoMap , nil
}
downloadSuccessful , retrievedReportsInfoMap , err := getDownloadedReportsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errB }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
return true , connectionIdentifier , retrievedReportsInfoMap , minimumRange , maximumRange , nil
}
// We will retrieve reports based on message inbox range
anyIntersectionFound , intersectionRangeStart , intersectionRangeEnd , err := byteRange . GetInboxIntersectionRangeFromTwoRanges ( rangeToRetrieveStart , rangeToRetrieveEnd , theirInboxHostRangeStart , theirInboxHostRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( anyIntersectionFound == false ) {
// Host is not hosting any reports within the inboxes range that we are requesting
// Skip this host
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
estimatedNumItemsInIntersectionRange , err := byteRange . GetEstimatedInboxSubrangeQuantity ( theirInboxHostRangeStart , theirInboxHostRangeEnd , hostedMessagesQuantityInt64 , intersectionRangeStart , intersectionRangeEnd )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
subrangesToQueryList , err := byteRange . SplitInboxRangeIntoEqualSubranges ( intersectionRangeStart , intersectionRangeEnd , estimatedNumItemsInIntersectionRange , maximumMessageHashesPerRequest )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
if ( hostProfileExists == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
if ( connectionEstablished == false ) {
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
//Outputs:
// -bool: Download successful
// -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported message hash)
// -error
getDownloadedReportsInfoMap := func ( ) ( bool , map [ [ 30 ] byte ] [ 26 ] byte , error ) {
retrievedReportsInfoMap := make ( map [ [ 30 ] byte ] [ 26 ] byte )
for _ , subrangeObject := range subrangesToQueryList {
subrangeStart := subrangeObject . SubrangeStart
subrangeEnd := subrangeObject . SubrangeEnd
emptyList := make ( [ ] [ 26 ] byte , 0 )
downloadSuccessful , currentRetrievedReportsInfoMap , err := sendRequests . GetMessageReportsInfoFromHost ( connectionIdentifier , hostIdentityHash , networkType , subrangeStart , subrangeEnd , emptyList )
if ( err != nil ) { return false , nil , err }
if ( downloadSuccessful == false ) {
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop downloading from host
err := logger . AddLogEntry ( "Network" , "Failed to download message reports info from host." )
if ( err != nil ) { return false , nil , err }
return false , nil , nil
}
for reportHash , reportedHash := range currentRetrievedReportsInfoMap {
retrievedReportsInfoMap [ reportHash ] = reportedHash
}
}
return true , retrievedReportsInfoMap , nil
}
downloadSuccessful , retrievedReportsInfoMap , err := getDownloadedReportsInfoMap ( )
if ( err != nil ) {
errB := peerClient . CloseConnection ( connectionIdentifier )
if ( errB != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , errB }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err
}
if ( downloadSuccessful == false ) {
err := peerClient . CloseConnection ( connectionIdentifier )
if ( err != nil ) { return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , err }
return false , [ 21 ] byte { } , nil , [ 10 ] byte { } , [ 10 ] byte { } , nil
}
return true , connectionIdentifier , retrievedReportsInfoMap , intersectionRangeStart , intersectionRangeEnd , nil
}
hostDownloadSuccessful , connectionIdentifier , retrievedReportsInfoMap , expectedRangeStart , expectedRangeEnd , err := getRetrievedReportsInfoMap ( )
if ( err != nil ) { return false , err }
if ( hostDownloadSuccessful == false ) {
// We did not download any reports from this host. Skip them.
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
// Now we see if we have any of these reports, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// We still download the reports to prevent host from knowing that we have those reports downloaded
// We then mark the host as malicious
reportHashesToDownloadList := make ( [ ] [ 30 ] byte , 0 )
hostIsMalicious := false
for receivedReportHash , receivedReportedHash := range retrievedReportsInfoMap {
//TODO: Disable this step if only 1 of Host/Moderator mode is enabled
storedReportExists , storedReportBytes , err := badgerDatabase . GetReport ( receivedReportHash )
if ( err != nil ) { return false , err }
checkIfHostIsOfferingInvalidReport := func ( ) ( bool , error ) {
if ( storedReportExists == true ) {
ableToRead , storedReportHash , _ , reportNetworkType , _ , storedReportType , storedReportReportedHash , _ , err := readReports . ReadReportAndHash ( false , storedReportBytes )
if ( err != nil ) { return false , err }
if ( ableToRead == false ) {
return false , errors . New ( "Database corrupt: Contains invalid report" )
}
if ( storedReportHash != receivedReportHash ) {
return false , errors . New ( "Database corrupt: Contains report with invalid entry key hash" )
}
if ( reportNetworkType != networkType ) {
return true , nil
}
areEqual := bytes . Equal ( storedReportReportedHash , receivedReportedHash [ : ] )
if ( areEqual == false ) {
return true , nil
}
if ( storedReportType != "Message" ) {
return true , nil
}
}
// Now we see if we have the reviewed message metadata
metadataExists , _ , messageNetworkType , _ , reportedMessageInbox , _ , err := contentMetadata . GetMessageMetadata ( receivedReportedHash )
if ( err != nil ) { return false , err }
if ( metadataExists == false ) {
return false , nil
}
if ( messageNetworkType != networkType ) {
// Report is reporting a message on a different networkType than the report
// The host should have checked for this already
// Host must be malicious
return true , nil
}
inboxIsWithinRange , err := byteRange . CheckIfInboxIsWithinRange ( expectedRangeStart , expectedRangeEnd , reportedMessageInbox )
if ( err != nil ) { return false , err }
if ( inboxIsWithinRange == false ) {
return true , nil
}
return false , nil
}
hostIsOfferingInvalidReport , err := checkIfHostIsOfferingInvalidReport ( )
if ( err != nil ) { return false , err }
if ( hostIsOfferingInvalidReport == true ) {
// Host is malicious, we must still download the report
hostIsMalicious = true
reportHashesToDownloadList = append ( reportHashesToDownloadList , receivedReportHash )
continue
}
if ( storedReportExists == true ) {
// We already have the report, skip downloading it
continue
}
// Report does not exist, we must check if we want to download it
// This works differently for Hosting and Moderation
// For instance, if the user is downloading reports for moderation, they will not redownload reports for messages that they have already reviewed
shouldDownload , err := checkIfReportShouldBeDownloaded ( receivedReportHash , receivedReportedHash )
if ( err != nil ) { return false , err }
if ( shouldDownload == true ) {
reportHashesToDownloadList = append ( reportHashesToDownloadList , receivedReportHash )
continue
}
}
if ( len ( reportHashesToDownloadList ) == 0 ) {
// Nothing to download, done with this host.
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
return true , nil
}
// Now we download reports
// We convert the [26]byte value to []byte to supply to GetReportsFromHost
reportsInfoMapForRequest := make ( map [ [ 30 ] byte ] [ ] byte )
for reportHash , reportedMessageHash := range retrievedReportsInfoMap {
reportsInfoMapForRequest [ reportHash ] = reportedMessageHash [ : ]
}
maximumReportsInResponse := serverResponse . MaximumReportsInResponse_GetReports
reportHashesSublistsList , err := helpers . SplitListIntoSublists ( reportHashesToDownloadList , maximumReportsInResponse )
if ( err != nil ) { return false , err }
numberOfDownloadedReports := 0
for _ , reportHashesSublist := range reportHashesSublistsList {
downloadSuccessful , reportsList , err := sendRequests . GetReportsFromHost ( connectionIdentifier , hostIdentityHash , networkType , reportHashesSublist , reportsInfoMapForRequest )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// Request has failed
// We will stop downloading from this host
break
}
for _ , reportBytes := range reportsList {
reportIsWellFormed , err := reportStorage . AddReport ( reportBytes )
if ( err != nil ) { return false , err }
if ( reportIsWellFormed == false ) {
return false , errors . New ( "GetReportsFromHost not verifying reports are well formed." )
}
}
numberOfDownloadedReports += len ( reportsList )
}
if ( hostIsMalicious == true ) {
err := maliciousHosts . AddHostToMaliciousHostsList ( hostIdentityHash )
if ( err != nil ) { return false , err }
}
if ( numberOfDownloadedReports != 0 ) {
numberOfReportsDownloadedString := helpers . ConvertIntToString ( numberOfDownloadedReports )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + numberOfReportsDownloadedString + " reports from host." )
if ( err != nil ) { return false , err }
}
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadReportsFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
//Inputs:
// -bool: Allow clearnet
// -[]string: List of identity hashes to retrieve viewable consensus statuses for
// -map[[28]byte][16]byte: Map of Profile Hash -> Profile Identity Hash
// Outputs:
// -error
func DownloadViewableStatusesFromHosts (
allowClearnet bool ,
networkType byte ,
knownOrUnknown string ,
identityHashesToRetrieveList [ ] [ 16 ] byte ,
profileHashesToRetrieveMap map [ [ 28 ] byte ] [ 16 ] byte ,
numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadViewableStatusesFromHosts called with invalid networkType: " + networkTypeString )
}
if ( knownOrUnknown != "Known" && knownOrUnknown != "Unknown" ) {
return errors . New ( "DownloadViewableStatusesFromHosts called with invalid knownOrUnknown: " + knownOrUnknown )
}
//Outputs:
// -bool: Downloaded any statuses
// -error
downloadViewableStatusesFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
exists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( exists == false ) {
// Host profile was deleted, skip host
return false , nil
}
hostIsHostingMateProfiles , hostMateRangeStart , hostMateRangeEnd , err := hostRanges . GetHostedIdentityHashRangeFromHostRawProfileMap ( hostRawProfileMap , "Mate" )
if ( err != nil ) { return false , err }
hostIsHostingHostProfiles , _ , _ , err := hostRanges . GetHostedIdentityHashRangeFromHostRawProfileMap ( hostRawProfileMap , "Host" )
if ( err != nil ) { return false , err }
hostIsHostingModeratorProfiles , _ , _ , err := hostRanges . GetHostedIdentityHashRangeFromHostRawProfileMap ( hostRawProfileMap , "Moderator" )
if ( err != nil ) { return false , err }
checkIfHostIsHostingIdentityHash := func ( identityHash [ 16 ] byte ) ( bool , error ) {
identityType , err := identity . GetIdentityTypeFromIdentityHash ( identityHash )
if ( err != nil ) { return false , err }
if ( identityType == "Mate" ) {
if ( hostIsHostingMateProfiles == false ) {
return false , nil
}
isWithinTheirRange , err := byteRange . CheckIfIdentityHashIsWithinRange ( hostMateRangeStart , hostMateRangeEnd , identityHash )
if ( err != nil ) { return false , err }
if ( isWithinTheirRange == false ) {
return false , nil
}
return true , nil
}
if ( identityType == "Host" ) {
if ( hostIsHostingHostProfiles == false ) {
return false , nil
}
return true , nil
}
// identityType == "Moderator"
if ( hostIsHostingModeratorProfiles == false ) {
return false , nil
}
return true , nil
}
relevantIdentityHashesToRetrieveList := make ( [ ] [ 16 ] byte , 0 )
for _ , identityHash := range identityHashesToRetrieveList {
isHostingIdentity , err := checkIfHostIsHostingIdentityHash ( identityHash )
if ( err != nil ) { return false , err }
if ( isHostingIdentity == false ) {
continue
}
statusIsKnown , _ , queriedHostsList , err := trustedViewableStatus . GetTrustedIdentityIsViewableStatus ( identityHash , networkType )
if ( err != nil ) { return false , err }
if ( knownOrUnknown == "Known" ) {
// We only want to retrieve known statuses. We dont need to avoid already queried hosts.
if ( statusIsKnown == false ) {
continue
}
} else {
// We only want to retrieve unknown statuses
if ( statusIsKnown == true ) {
continue
}
// We must avoid hosts we have already queried
alreadyQueried := slices . Contains ( queriedHostsList , hostIdentityHash )
if ( alreadyQueried == true ) {
continue
}
}
relevantIdentityHashesToRetrieveList = append ( relevantIdentityHashesToRetrieveList , identityHash )
}
relevantProfileHashesToRetrieveList := make ( [ ] [ 28 ] byte , 0 )
for profileHash , profileIdentityHash := range profileHashesToRetrieveMap {
isHostingIdentity , err := checkIfHostIsHostingIdentityHash ( profileIdentityHash )
if ( err != nil ) { return false , err }
if ( isHostingIdentity == false ) {
continue
}
statusIsKnown , _ , queriedHostsList , err := trustedViewableStatus . GetTrustedProfileIsViewableStatus ( profileHash )
if ( err != nil ) { return false , err }
if ( knownOrUnknown == "Known" ) {
// We only want to retrieve known statuses. We dont need to avoid already queried hosts.
if ( statusIsKnown == false ) {
continue
}
} else {
// We only want to retrieve unknown statuses
if ( statusIsKnown == true ) {
continue
}
// We must avoid hosts we have already queried
alreadyQueried := slices . Contains ( queriedHostsList , hostIdentityHash )
if ( alreadyQueried == true ) {
continue
}
}
relevantProfileHashesToRetrieveList = append ( relevantProfileHashesToRetrieveList , profileHash )
}
if ( len ( relevantIdentityHashesToRetrieveList ) == 0 && len ( relevantProfileHashesToRetrieveList ) == 0 ) {
// Nothing to retrieve from this host. Skip them.
return false , nil
}
// Now we must split up our request to not exceed maximum size
// The requests/responses for identity hashes will always be smaller than profile hashes, for the same number of identities/profiles
// This is because profile hashes are larger than identity hashes
// We will split request so that we can fit the maximum amount of identity/profile hashes in all of the requests
numberOfIdentityHashesToRetrieve := len ( relevantIdentityHashesToRetrieveList )
numberOfProfileHashesToRetrieve := len ( relevantProfileHashesToRetrieveList )
maximumIdentityHashesInResponse := serverResponse . MaximumIdentitiesInResponse_GetIdentityViewableStatuses
maximumProfileHashesInResponse := serverResponse . MaximumProfilesInResponse_GetProfileViewableStatuses
numberOfRequestsNeededForIdentityHashes := float64 ( numberOfIdentityHashesToRetrieve ) / float64 ( maximumIdentityHashesInResponse )
numberOfRequestsNeededForProfileHashes := float64 ( numberOfProfileHashesToRetrieve ) / float64 ( maximumProfileHashesInResponse )
totalNumberOfRequestsNeeded , err := helpers . CeilFloat64ToInt64 ( numberOfRequestsNeededForIdentityHashes + numberOfRequestsNeededForProfileHashes )
if ( err != nil ) { return false , err }
numberOfIdentityHashesPerRequest := int64 ( numberOfIdentityHashesToRetrieve ) / totalNumberOfRequestsNeeded
numberOfProfileHashesPerRequest := int64 ( numberOfProfileHashesToRetrieve ) / totalNumberOfRequestsNeeded
identityHashesSublistsList , err := helpers . SplitListIntoSublists ( relevantIdentityHashesToRetrieveList , int ( numberOfIdentityHashesPerRequest ) )
if ( err != nil ) { return false , err }
profileHashesSublistsList , err := helpers . SplitListIntoSublists ( relevantProfileHashesToRetrieveList , int ( numberOfProfileHashesPerRequest ) )
if ( err != nil ) { return false , err }
identityHashesSublistsListLength := len ( identityHashesSublistsList )
profileHashesSublistsListLength := len ( profileHashesSublistsList )
getLongerListLength := func ( ) int {
if ( identityHashesSublistsListLength > profileHashesSublistsListLength ) {
return identityHashesSublistsListLength
}
return profileHashesSublistsListLength
}
longerListLength := getLongerListLength ( )
totalDownloadedStatuses := 0
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( hostProfileExists == false ) {
return false , nil
}
if ( connectionEstablished == false ) {
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
for i := 0 ; i < longerListLength ; i ++ {
getIdentityHashesSublist := func ( ) [ ] [ 16 ] byte {
if ( i < identityHashesSublistsListLength ) {
return identityHashesSublistsList [ i ]
}
emptyList := make ( [ ] [ 16 ] byte , 0 )
return emptyList
}
getProfileHashesSublist := func ( ) [ ] [ 28 ] byte {
if ( i < profileHashesSublistsListLength ) {
return profileHashesSublistsList [ i ]
}
emptyList := make ( [ ] [ 28 ] byte , 0 )
return emptyList
}
identityHashesSublist := getIdentityHashesSublist ( )
profileHashesSublist := getProfileHashesSublist ( )
downloadSuccessful , identityHashStatusesMap , profileHashStatusesMap , err := sendRequests . GetViewableStatusesFromHost ( connectionIdentifier , hostIdentityHash , networkType , identityHashesSublist , profileHashesSublist )
if ( err != nil ) { return false , err }
if ( downloadSuccessful == false ) {
// This host has failed to respond. We will stop downloading from them.
return false , nil
}
for identityHash , viewableStatus := range identityHashStatusesMap {
err := trustedViewableStatus . AddTrustedIdentityIsViewableStatus ( identityHash , hostIdentityHash , networkType , viewableStatus )
if ( err != nil ) { return false , err }
totalDownloadedStatuses += 1
}
for profileHash , viewableStatus := range profileHashStatusesMap {
err := trustedViewableStatus . AddTrustedProfileIsViewableStatus ( profileHash , hostIdentityHash , viewableStatus )
if ( err != nil ) { return false , err }
profileIdentityHash , exists := profileHashesToRetrieveMap [ profileHash ]
if ( exists == false ) {
return false , errors . New ( "profileHashesToRetrieveMap missing profileHash" )
}
if ( viewableStatus == true ) {
// The host will only indicate a viewable profile if they know that the identity is not banned
err := trustedViewableStatus . AddTrustedIdentityIsViewableStatus ( profileIdentityHash , hostIdentityHash , networkType , true )
if ( err != nil ) { return false , err }
}
totalDownloadedStatuses += 1
}
}
totalDownloadedStatusesString := helpers . ConvertIntToString ( totalDownloadedStatuses )
err = logger . AddLogEntry ( "Network" , "Successfully downloaded " + totalDownloadedStatusesString + " viewable statuses from host." )
if ( err != nil ) { return false , err }
return true , nil
}
eligibleHostsList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
queriedHosts := 0
for _ , hostIdentityHash := range eligibleHostsList {
successfulDownload , err := downloadViewableStatusesFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( successfulDownload == true ) {
queriedHosts += 1
}
if ( queriedHosts >= numberOfHostsToQuery ) {
return nil
}
}
return nil
}
func DownloadModeratorAddressDepositsFromHosts ( allowClearnet bool , networkType byte , cryptocurrencyName string , knownOrUnknown string , numberOfHostsToQuery int ) error {
isValid := helpers . VerifyNetworkType ( networkType )
if ( isValid == false ) {
networkTypeString := helpers . ConvertByteToString ( networkType )
return errors . New ( "DownloadModeratorAddressDepositsFromHosts called with invalid networkType: " + networkTypeString )
}
if ( cryptocurrencyName != "Ethereum" && cryptocurrencyName != "Cardano" ) {
return errors . New ( "DownloadModeratorAddressDepositsFromHosts called with invalid cryptocurrencyName: " + cryptocurrencyName )
}
if ( knownOrUnknown != "Unknown" && knownOrUnknown != "Known" ) {
return errors . New ( "DownloadModeratorAddressDepositsFromHosts called with invalid knownOrUnknown: " + knownOrUnknown )
}
//TODO: Check if local Ethereum/Cardano node is being used (either here or in function that calls this function)
//TODO: Check to see if less than 3 blockchain hosts exist, in which case, relax the required rule of 3 needed
// We will first see which identity deposits are known/unknown
//Outputs:
// -bool: Downloaded any deposits
// -error
downloadDepositsFromHost := func ( hostIdentityHash [ 16 ] byte ) ( bool , error ) {
profileExists , _ , _ , _ , _ , hostRawProfileMap , err := profileStorage . GetNewestUserProfile ( hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( profileExists == false ) {
return false , nil
}
exists , isHostingBlockchain , err := readProfiles . GetFormattedProfileAttributeFromRawProfileMap ( hostRawProfileMap , "Hosting" + cryptocurrencyName + "Blockchain" )
if ( err != nil ) { return false , err }
if ( exists == false || isHostingBlockchain != "Yes" ) {
return false , nil
}
localBlockchainNodeEnabled , addressesToUpdateList , err := moderatorScores . GetModeratorCryptoDepositAddressesList ( cryptocurrencyName , knownOrUnknown )
if ( err != nil ) { return false , err }
if ( localBlockchainNodeEnabled == true ) {
// We must have enabled the node after our earlier check
// Skip all remaining hosts.
return false , nil
}
if ( len ( addressesToUpdateList ) == 0 ) {
// No addresses to update remain.
return false , nil
}
//TODO: Add limit to size of below
hostProfileExists , connectionEstablished , connectionIdentifier , err := peerClient . EstablishNewConnectionToHost ( allowClearnet , hostIdentityHash , networkType )
if ( err != nil ) { return false , err }
if ( hostProfileExists == false ) {
return false , nil
}
if ( connectionEstablished == false ) {
return false , nil
}
defer peerClient . CloseConnection ( connectionIdentifier )
successfulDownload , retrievedDepositObjectsList , addressesWithNoDepositsList , err := sendRequests . GetAddressDepositsFromHost ( connectionIdentifier , hostIdentityHash , networkType , cryptocurrencyName , addressesToUpdateList )
if ( err != nil ) { return false , err }
if ( successfulDownload == false ) {
err := logger . AddLogEntry ( "Network" , "Failed to retrieve address deposits from host." )
if ( err != nil ) { return false , err }
// Skip to next host
return false , nil
}
err = trustedAddressDeposits . AddAddressDepositObjectsListToCache ( hostIdentityHash , cryptocurrencyName , retrievedDepositObjectsList )
if ( err != nil ) { return false , err }
err = trustedAddressDeposits . AddAddressesWithNoDepositsToCache ( hostIdentityHash , cryptocurrencyName , addressesWithNoDepositsList )
if ( err != nil ) { return false , err }
return true , nil
}
// We query each host for the identity deposits we want
// After each host, we get the new known/unknown identities list to make
// sure we do not query deposits which were unknown but have become known
hostsToQueryList , err := eligibleHosts . GetEligibleHostsList ( networkType )
if ( err != nil ) { return err }
hostsQueried := 0
for _ , hostIdentityHash := range hostsToQueryList {
downloadedAnyDeposits , err := downloadDepositsFromHost ( hostIdentityHash )
if ( err != nil ) { return err }
if ( downloadedAnyDeposits == true ) {
hostsQueried += 1
}
if ( hostsQueried >= numberOfHostsToQuery ) {
break
}
}
return nil
}