sarasova/seekia
sarasova/seekia
1
1
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Activity
seekia/internal/network/queryHosts/queryHosts.go

3372 lines
133 KiB
Go
Raw Blame History

// queryHosts provides functions to download information from hosts
// These include parameters, profiles, messages, reports, reviews, identity deposits, and trusted moderation viewable statuses
// Each function will query a specified number of eligible hosts
package queryHosts
//TODO: If you download a user's identity, then the host is saying that the user's identity is funded. Add it to trustedFundedStatus
//TODO: Split requests up so that memory does not have to store all of the requested hashes
//TODO: Set a maximum limit, so that only a certain amount of data will be requested from each host
//TODO: Choose hosts more intelligently based on what ranges we are requesting
// Right now, we are sending requests to randomly chosen hosts until we reach the numberOfHostsToQuery
// This is a bad strategy, because, if we are requesting range 20-100, we might request from a host that is only hosting from range 10-25
// We should use numberOfHostsToQuery to choose a set of hosts that will host the entire range
// We must make sure that we sometimes request from hosts that only host a small number of total profiles.
// TODO: Add BroadcastToHosts, GetFundedStatuses, and more
//TODO: If only Host/Moderator mode is enabled, we can skip all of the downloaded identity/inbox fingerprinting prevention logic
// This logic is designed to prevent hosts from learning the user's moderator/host identity/behavior from their downloaded content
// For example, a user is moderating and hosting. requestors could determine a host's moderator identity by querying for
// profiles outside of their host range and seeing which mate profiles they have downloaded.
// Using this information, they could learn their moderator range and link it to their moderator identity/profile.
//
// We cannot disable fingerprinting if Mate mode is enabled, because a user's Mate criteria will change, so we cannot allow the host to know what their criteria was previously
// This is also true if a Host or Moderator's ranges change, but in those cases, the leak of privacy is fine
// This is because the information (previous host/moderator ranges) is not sensitive, and not a problem if it is leaked
import "seekia/internal/byteRange"
import "seekia/internal/badgerDatabase"
import "seekia/internal/contentMetadata"
import "seekia/internal/encoding"
import "seekia/internal/helpers"
import "seekia/internal/identity"
import "seekia/internal/logger"
import "seekia/internal/messaging/chatMessageStorage"
import "seekia/internal/moderation/moderatorScores"
import "seekia/internal/moderation/readReports"
import "seekia/internal/moderation/readReviews"
import "seekia/internal/moderation/reportStorage"
import "seekia/internal/moderation/reviewStorage"
import "seekia/internal/moderation/trustedAddressDeposits"
import "seekia/internal/moderation/trustedViewableStatus"
import "seekia/internal/network/eligibleHosts"
import "seekia/internal/network/hostRanges"
import "seekia/internal/network/maliciousHosts"
import "seekia/internal/network/mateCriteria"
import "seekia/internal/network/peerClient"
import "seekia/internal/network/sendRequests"
import "seekia/internal/network/serverResponse"
import "seekia/internal/parameters/parametersStorage"
import "seekia/internal/profiles/profileStorage"
import "seekia/internal/profiles/readProfiles"
import "bytes"
import "errors"
import "slices"
func DownloadParametersFromHosts(allowClearnet bool, networkType byte, numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadParametersFromHosts called with invalid networkType: " + networkTypeString)
}
// Outputs:
// -bool: Successful download from host
// -error
downloadParametersFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
exists, hostIsHostingParameters, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "HostingParameters")
if (err != nil) { return false, err }
if (exists == false){
return false, errors.New("Database corrupt: Contains host Profile missing HostingParameters")
}
if (hostIsHostingParameters != "Yes"){
// This host is not hosting parameters, skip
return false, nil
}
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, err }
if (hostProfileExists == false){
return false, nil
}
if (connectionEstablished == false){
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
downloadSuccessful, parametersInfoMap, err := sendRequests.GetParametersInfoFromHost(connectionIdentifier, hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
return false, nil
}
if (len(parametersInfoMap) == 0){
// The host has provided no parameters. We will not consider this a successful download
return false, nil
}
// We will check to see if our parameters broadcast time is older than the one the host is offering
// If it is, we will download it and continue
parametersTypesToRetrieveList := make([]string, 0)
for parametersType, parametersBroadcastTime := range parametersInfoMap{
storedParametersFound, _, _, storedParametersBroadcastTime, err := parametersStorage.GetAuthorizedParameters(parametersType, networkType)
if (err != nil) { return false, err }
if (storedParametersFound == true){
if (storedParametersBroadcastTime < parametersBroadcastTime){
// The offered parameters are not newer than our existing parameters
// Skip downloading them.
continue
}
}
parametersTypesToRetrieveList = append(parametersTypesToRetrieveList, parametersType)
}
if (len(parametersTypesToRetrieveList) == 0){
// The host has no parameters that we need. Nothing to download.
return true, nil
}
// Now we download parameters
downloadSuccessful, receivedParametersList, err := sendRequests.GetParametersFromHost(connectionIdentifier, hostIdentityHash, networkType, parametersTypesToRetrieveList)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Unable to download parameters. Nothing left to do with this host.
return false, nil
}
if (len(receivedParametersList) == 0){
// Host is probably malicious
// Host could have received new admin permissions after we got parameters info
// Either way, we do not count this as a successful download
return false, nil
}
for _, parametersBytes := range receivedParametersList{
parametersAreValid, _, err := parametersStorage.AddParameters(parametersBytes)
if (err != nil) { return false, err }
if (parametersAreValid == false){
return false, errors.New("GetParametersFromHost not verifying parameters.")
}
}
// These parameters were downloaded, not necessarily authorized or added
numberOfDownloadedParametersString := helpers.ConvertIntToString(len(receivedParametersList))
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfDownloadedParametersString + " network parameters from host.")
if (err != nil) { return false, err }
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadParametersFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadProfilesFromHosts(
allowClearnet bool,
networkType byte,
profileTypeToRetrieve string,
rangeToRetrieveStart [16]byte,
rangeToRetrieveEnd [16]byte,
identityHashesToRetrieveList [][16]byte,
criteria []byte,
getNewestProfilesOnly bool,
getViewableProfilesOnly bool,
downloadAllMode bool,
checkIfProfileShouldBeDownloaded func([28]byte, [16]byte, int64)(bool, error),
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadProfilesFromHosts called with invalid networkType: " + networkTypeString)
}
// We cycle through each host
// We get the profiles info for profiles that are within our requested criteria/range/identitites list
// We see if any are knowingly outside of criteria, and download anyway if they are (to resist fingerprinting)
// We download the profiles.
// We don't download any profiles that we already have (if desiresPruningMode == false)
//TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable profiles
//Outputs:
// -bool: Successfully downloaded from host (false if host does not have any profiles we need)
// -error
downloadProfilesFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
hostIsHostingProfileType, theirHostRangeStart, theirHostRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, profileTypeToRetrieve)
if (err != nil) { return false, err }
if (hostIsHostingProfileType == false){
// Host is not hosting any profiles of this profileType. Skip to next host
return false, nil
}
//Outputs:
// -bool: Download successful
// -[21]byte: Connection identifier
// -[]serverResponse.ProfileInfoStruct: Retrieved profiles info objects list
// -error
getRetrievedProfileInfoObjectsList := func()(bool, [21]byte, []serverResponse.ProfileInfoStruct, error){
// Our request will either be based on identity hash range or identity hash list
// We will split up the request into either ranges or identity hashes
maximumProfilesInGetProfilesInfoRequest := serverResponse.MaximumProfilesInResponse_GetProfilesInfo - 100
if (len(identityHashesToRetrieveList) != 0){
// We will retrieve identities lists, not identity ranges
anyValuesExist, identitiesInMyRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identityHashesToRetrieveList)
if (err != nil) { return false, [21]byte{}, nil, err }
if (anyValuesExist == false){
// This should not happen
return false, [21]byte{}, nil, errors.New("Identity hashes list to retrieve contains no identities within request range.")
}
anyValuesExist, identitiesInTheirRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostRangeStart, theirHostRangeEnd, identitiesInMyRangeList)
if (err != nil) { return false, [21]byte{}, nil, err }
if (anyValuesExist == false){
// This host is not hosting any identities we desire. We will skip them.
return false, [21]byte{}, nil, nil
}
// Now we split request into sublists if needed
getMaximumIdentitiesPerRequest := func()int{
if (getNewestProfilesOnly == true){
return maximumProfilesInGetProfilesInfoRequest
}
// We assume each identity has approximately 3 profiles
maximumIdentitiesPerRequest := maximumProfilesInGetProfilesInfoRequest / 3
return maximumIdentitiesPerRequest
}
maximumIdentitiesPerRequest := getMaximumIdentitiesPerRequest()
sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInTheirRangeList, maximumIdentitiesPerRequest)
if (err != nil) { return false, [21]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, nil
}
downloadProfileInfoObjectsList := func()(bool, []serverResponse.ProfileInfoStruct, error){
retrievedProfileInfoObjectsList := make([]serverResponse.ProfileInfoStruct, 0)
minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds()
for _, identityHashesSublist := range sublistsToQueryList{
downloadSuccessful, profilesInfoObjectsList, err := sendRequests.GetProfilesInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist, criteria, getNewestProfilesOnly, getViewableProfilesOnly)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this subrange.
// Could be that host went offline permanently or just this connection failed.
// Either way, cease connection to host
err := logger.AddLogEntry("Network", "Failed to download profiles info from host.")
if (err != nil) { return false, nil, err }
err = peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, nil, err }
return false, nil, nil
}
retrievedProfileInfoObjectsList = append(retrievedProfileInfoObjectsList, profilesInfoObjectsList...)
}
return true, retrievedProfileInfoObjectsList, nil
}
downloadSuccessful, retrievedProfileInfoObjectsList, err := downloadProfileInfoObjectsList()
if (err != nil){
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, errB }
return false, [21]byte{}, nil, err
}
if (downloadSuccessful == false){
err = peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, err }
return false, [21]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedProfileInfoObjectsList, nil
}
// We will retrieve identity ranges, not identity lists
//Outputs:
// -Any intersection found
// -[16]byte: Intersection range start
// -[16]byte: Intersection range end
// -error
getIntersectionRange := func()(bool, [16]byte, [16]byte, error){
if (profileTypeToRetrieve != "Mate"){
// Hosts must host all or none of host/moderator identities, so intersection is always all our requested identities
return true, rangeToRetrieveStart, rangeToRetrieveEnd, nil
}
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostRangeStart, theirHostRangeEnd)
if (err != nil) { return false, [16]byte{}, [16]byte{}, err }
if (anyIntersectionFound == false){
return false, [16]byte{}, [16]byte{}, nil
}
return true, intersectionRangeStart, intersectionRangeEnd, nil
}
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := getIntersectionRange()
if (err != nil) { return false, [21]byte{}, nil, err }
if (anyIntersectionFound == false){
// Host is not hosting any profiles within the range that we are requesting
// Skip this host
return false, [21]byte{}, nil, nil
}
exists, hostedProfilesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, profileTypeToRetrieve + "ProfilesQuantity")
if (err != nil) { return false, [21]byte{}, nil, err }
if (exists == false){
return false, [21]byte{}, nil, errors.New("Database corrupt: Contains host profile missing profilesQuantity")
}
hostedProfilesQuantityInt64, err := helpers.ConvertStringToInt64(hostedProfilesQuantity)
if (err != nil) { return false, [21]byte{}, nil, err }
if (hostedProfilesQuantityInt64 == 0){
// Host must have just started their node. Skip them.
return false, [21]byte{}, nil, nil
}
estimatedNumItemsInSubrange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostRangeStart, theirHostRangeEnd, hostedProfilesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, err }
subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInSubrange, int64(maximumProfilesInGetProfilesInfoRequest))
if (err != nil) { return false, [21]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, nil
}
//Outputs:
// -bool: Download successful
// -[]serverResponse.ProfileInfoStruct: Retrieved profiles info map list
// -error
downloadProfileInfoObjectsList := func()(bool, []serverResponse.ProfileInfoStruct, error){
retrievedProfileInfoObjectsList := make([]serverResponse.ProfileInfoStruct, 0)
for _, subrangeObject := range subrangesToQueryList{
subrangeStart := subrangeObject.SubrangeStart
subrangeEnd := subrangeObject.SubrangeEnd
downloadSuccessful, profileInfoObjectsList, err := sendRequests.GetProfilesInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, subrangeStart, subrangeEnd, identityHashesToRetrieveList, criteria, getNewestProfilesOnly, getViewableProfilesOnly)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this subrange.
// Could be that host went offline permanently or just this connection failed.
// Either way, close connection
err := logger.AddLogEntry("Network", "Failed to download profiles info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
retrievedProfileInfoObjectsList = append(retrievedProfileInfoObjectsList, profileInfoObjectsList...)
}
return true, retrievedProfileInfoObjectsList, nil
}
downloadSuccessful, retrievedProfileInfoObjectsList, err := downloadProfileInfoObjectsList()
if (err != nil){
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, errB }
return false, [21]byte{}, nil, err
}
if (downloadSuccessful == false){
err = peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, err }
return false, [21]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedProfileInfoObjectsList, nil
}
downloadSuccessful, connectionIdentifier, retrievedProfilesInfoObjectsList, err := getRetrievedProfileInfoObjectsList()
if (err != nil) { return false, err }
if (downloadSuccessful == false){
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
if (len(retrievedProfilesInfoObjectsList) == 0){
// Host has no profiles that we need. We are done with this host.
return true, nil
}
profileHashesToDownloadList := make([][28]byte, 0)
// Structure: Profile Hash -> Expected Broadcast Time
profileBroadcastTimesMap := make(map[[28]byte]int64)
// Structure: Profile Hash -> Identity Hash
profileIdentityHashesMap := make(map[[28]byte][16]byte)
hostIsMalicious := false
for _, profileInfoObject := range retrievedProfilesInfoObjectsList{
receivedProfileHash := profileInfoObject.ProfileHash
receivedProfileAuthor := profileInfoObject.ProfileAuthor
receivedProfileBroadcastTime := profileInfoObject.ProfileBroadcastTime
// Now we check to see if we already have the profile
// We check to see if it is known to be outside of requested range/identities list/criteria (host is malicious)
// If it is, we still request it in our download, and then set host as malicious afterwards
// Otherwise, host would know that we have this profile stored
//TODO: Disable this step if only host/moderator mode is enabled
storedProfileExists, storedProfileBytes, err := badgerDatabase.GetUserProfile(profileTypeToRetrieve, receivedProfileHash)
if (err != nil) { return false, err }
checkIfHostIsOfferingInvalidProfile := func()(bool, error){
if (storedProfileExists == false){
return false, nil
}
ableToRead, storedProfileHash, profileVersion, profileNetworkType, storedProfileAuthor, storedProfileBroadcastTime, _, storedRawProfileMap, err := readProfiles.ReadProfileAndHash(false, storedProfileBytes)
if (err != nil) { return false, err }
if (ableToRead == false){
return false, errors.New("Database corrupt: Contains invalid profile.")
}
if (storedProfileHash != receivedProfileHash){
return false, errors.New("Database corrupt: Contains profile with different hash than entry key.")
}
if (profileNetworkType != networkType){
return true, nil
}
if (storedProfileAuthor != receivedProfileAuthor){
return true, nil
}
if (storedProfileBroadcastTime != receivedProfileBroadcastTime){
return true, nil
}
if (criteria != nil){
criteriaIsValid, fulfillsCriteria, err := mateCriteria.CheckIfMateProfileFulfillsCriteria(false, profileVersion, storedRawProfileMap, criteria)
if (err != nil) { return false, err }
if (criteriaIsValid == false){
return false, errors.New("DownloadProfilesFromHosts called with invalid criteria.")
}
if (fulfillsCriteria == false){
return true, nil
}
}
return false, nil
}
hostIsOfferingInvalidProfile, err := checkIfHostIsOfferingInvalidProfile()
if (err != nil) { return false, err }
if (hostIsOfferingInvalidProfile == true){
// Host is malicious, we must still download the profile
hostIsMalicious = true
profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash)
profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor
profileBroadcastTimesMap[receivedProfileHash] = receivedProfileBroadcastTime
continue
}
if (downloadAllMode == true){
// downloadAllMode is enabled, we must download all profiles regardless of if we already have it
// This is done to prevent hosts learning our private desires
profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash)
profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor
profileBroadcastTimesMap[receivedProfileHash] = receivedProfileBroadcastTime
continue
}
if (storedProfileExists == true){
// We already have profile, skip downloading it
if (getViewableProfilesOnly == true){
err = trustedViewableStatus.AddTrustedProfileIsViewableStatus(receivedProfileHash, hostIdentityHash, true)
if (err != nil) { return false, err }
}
continue
}
// Profile does not exist, we must check if we want to download it
// This works differently for Hosting, Moderation, and Browsing
shouldDownload, err := checkIfProfileShouldBeDownloaded(receivedProfileHash, receivedProfileAuthor, receivedProfileBroadcastTime)
if (err != nil) { return false, err }
if (shouldDownload == true){
profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash)
profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor
profileBroadcastTimesMap[receivedProfileHash] = receivedProfileBroadcastTime
continue
}
}
if (len(profileHashesToDownloadList) == 0){
// Nothing to download, done with this host.
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
return true, nil
}
// Now we retrieve profiles from host
maximumProfileHashesInResponse, err := serverResponse.GetMaximumProfilesInResponse_GetProfiles(profileTypeToRetrieve)
if (err != nil) { return false, err }
profileHashesSublistsList, err := helpers.SplitListIntoSublists(profileHashesToDownloadList, maximumProfileHashesInResponse)
if (err != nil) { return false, err }
numberOfDownloadedProfiles := 0
for _, profileHashesSublist := range profileHashesSublistsList{
downloadSuccessful, profilesList, err := sendRequests.GetProfilesFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, profileHashesSublist, profileIdentityHashesMap, profileBroadcastTimesMap, nil)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Request has failed
// We will stop downloading from this host
break
}
for _, profileBytes := range profilesList{
//TODO: If downloadAll mode is enabled, we need to check to see if profile should be imported or not (desires pruning mode)
profileIsWellFormed, addedProfileHash, err := profileStorage.AddUserProfile(profileBytes)
if (err != nil) { return false, err }
if (profileIsWellFormed == false){
return false, errors.New("GetProfilesFromHost not verifying profiles are well formed.")
}
if (getViewableProfilesOnly == true){
err = trustedViewableStatus.AddTrustedProfileIsViewableStatus(addedProfileHash, hostIdentityHash, true)
if (err != nil) { return false, err }
}
}
numberOfDownloadedProfiles += len(profilesList)
}
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
if (numberOfDownloadedProfiles != 0){
numberOfProfilesDownloadedString := helpers.ConvertIntToString(numberOfDownloadedProfiles)
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfProfilesDownloadedString + " " + profileTypeToRetrieve + " profiles from host.")
if (err != nil) { return false, err }
}
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadProfilesFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadMessagesFromHosts(
allowClearnet bool,
networkType byte,
rangeToRetrieveStart [10]byte,
rangeToRetrieveEnd [10]byte,
inboxesToRetrieveList [][10]byte,
getViewableMessagesOnly bool,
getDecryptableMessagesOnly bool,
checkIfMessageShouldBeDownloaded func([26]byte)(bool, error),
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadMessagesFromHosts called with invalid networkType: " + networkTypeString)
}
// We cycle through each host
// We get all message hashes that they host that are within our requested range/inboxes list
// We see if any are knowingly outside of inboxes list, and download anyway if they are (to resist fingerprinting)
// We download the messages that we don't already have
//TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable messages
//Outputs:
// -bool: Successfully downloaded from host (false if host does not have any messages we need)
// -error
downloadMessagesFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
hostIsHostingMessages, theirInboxRangeStart, theirInboxRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap)
if (hostIsHostingMessages == false){
// Host is not hosting any messages. Skip to next host
return false, nil
}
//Outputs:
// -bool: Download successful
// -[21]byte: Connection identifier
// -[][26]byte: Retrieved message hashes list
// -[10]byte: Inbox Range start of requested messages
// -[10]byte: Inbox range end of requested messages
// -[][10]byte: Requested inboxes list (nil if none were requested)
// -error
getReceivedMessageHashesList := func()(bool, [21]byte, [][26]byte, [10]byte, [10]byte, [][10]byte, error){
// Our request will either be based on inbox range or inbox list
// We will split up the request into either ranges or inboxes
exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity")
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (exists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile missing MessagesQuantity")
}
hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
maximumMessageHashesInResponse := serverResponse.MaximumMessageHashesInResponse_GetMessageHashesList - 100
if (len(inboxesToRetrieveList) != 0){
// We will split request into inbox lists, not inbox ranges
anyValuesExist, inboxesInMyRangeList, err := byteRange.GetAllInboxesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, inboxesToRetrieveList)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (anyValuesExist == false){
// This should not happen
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Inboxes list to retrieve contains no inboxes within request range.")
}
anyValuesExist, inboxesInTheirRangeList, err := byteRange.GetAllInboxesInListWithinRange(theirInboxRangeStart, theirInboxRangeEnd, inboxesInMyRangeList)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (anyValuesExist == false){
// This host is not hosting any inboxes we desire. We will skip them.
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
// Now we split request into sublists if needed
exists, hostedInboxesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "InboxesQuantity")
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (exists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile missing InboxesQuantity")
}
hostedInboxesQuantityInt64, err := helpers.ConvertStringToInt64(hostedInboxesQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid InboxesQuantity: " + hostedInboxesQuantity)
}
if (hostedInboxesQuantityInt64 == 0){
// Host has no inboxes
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
estimatedNumMessagesPerInbox := hostedMessagesQuantityInt64/hostedInboxesQuantityInt64
maximumInboxesPerRequest := int64(maximumMessageHashesInResponse) / estimatedNumMessagesPerInbox
maximumInboxesPerRequestInt, err := helpers.ConvertInt64ToInt(maximumInboxesPerRequest)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
sublistsToQueryList, err := helpers.SplitListIntoSublists(inboxesInTheirRangeList, maximumInboxesPerRequestInt)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds()
//Outputs:
// -bool: Download successful
// -[][26]byte: Message hashes list
// -error
getMessageHashesListFromHost := func()(bool, [][26]byte, error){
retrievedMessageHashesList := make([][26]byte, 0)
for _, inboxesSublist := range sublistsToQueryList{
downloadSuccessful, messageHashesList, err := sendRequests.GetMessageHashesListFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, inboxesSublist, getViewableMessagesOnly, getDecryptableMessagesOnly)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// We are done with this host
err := logger.AddLogEntry("Network", "Failed to download message hashes from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
retrievedMessageHashesList = append(retrievedMessageHashesList, messageHashesList...)
}
return true, retrievedMessageHashesList, nil
}
downloadSuccessful, retrievedMessageHashesList, err := getMessageHashesListFromHost()
if (err != nil){
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errB }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedMessageHashesList, minimumRange, maximumRange, inboxesInTheirRangeList, nil
}
// We will retrieve using inbox ranges, not inbox lists
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxRangeStart, theirInboxRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (anyIntersectionFound == false){
// Host is not hosting any messages within the inboxes range that we are requesting
// Skip this host
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxRangeStart, theirInboxRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, int64(maximumMessageHashesInResponse))
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
getMessageHashesListFromHost := func()(bool, [][26]byte, error){
retrievedMessageHashesList := make([][26]byte, 0)
for _, subrangeObject := range subrangesToQueryList{
subrangeStart := subrangeObject.SubrangeStart
subrangeEnd := subrangeObject.SubrangeEnd
downloadSuccessful, messageHashesList, err := sendRequests.GetMessageHashesListFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, inboxesToRetrieveList, getViewableMessagesOnly, getDecryptableMessagesOnly)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this subrange.
//Could be that host went offline permanently or just this connection failed.
// Either way, end connection to host
err := logger.AddLogEntry("Network", "Failed to download message hashes from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
retrievedMessageHashesList = append(retrievedMessageHashesList, messageHashesList...)
}
return true, retrievedMessageHashesList, nil
}
downloadSuccessful, retrievedMessageHashesList, err := getMessageHashesListFromHost()
if (err != nil){
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errB }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
}
emptyList := make([][10]byte, 0)
return true, connectionIdentifier, retrievedMessageHashesList, intersectionRangeStart, intersectionRangeEnd, emptyList, nil
}
downloadSuccessful, connectionIdentifier, retrievedMessageHashesList, requestedInboxesRangeStart, requestedInboxesRangeEnd, requestedInboxesList, err := getReceivedMessageHashesList()
if (err != nil) { return false, err }
if (downloadSuccessful == false){
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
// Now we see if we have any of these messages, and if they do not fulfill the request range/inboxes list
// If they do not fulfill the request, we know host is malicious
// We still download the messages to prevent host from knowing that we have those messages downloaded
// We then mark the host as malicious
//TODO: Immediately stop connection to host if only Host/Moderator mode is enabled, because fingerprinting is not a risk.
messageHashesToDownloadList := make([][26]byte, 0)
hostIsMalicious := false
for _, receivedMessageHash := range retrievedMessageHashesList{
checkIfHostIsOfferingInvalidMessage := func()(bool, error){
messageMetadataExists, _, messageNetworkType, _, messageInbox, _, err := contentMetadata.GetMessageMetadata(receivedMessageHash)
if (err != nil) { return false, err }
if (messageMetadataExists == false){
return false, nil
}
if (messageNetworkType != networkType){
return true, nil
}
if (len(requestedInboxesList) != 0){
isInInboxesList := slices.Contains(requestedInboxesList, messageInbox)
if (isInInboxesList == false){
return true, nil
}
} else {
inboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(requestedInboxesRangeStart, requestedInboxesRangeEnd, messageInbox)
if (err != nil) { return false, err }
if (inboxIsWithinRange == false){
return true, nil
}
}
return false, nil
}
hostIsOfferingInvalidMessage, err := checkIfHostIsOfferingInvalidMessage()
if (err != nil) { return false, err }
if (hostIsOfferingInvalidMessage == true){
// Host is malicious, we must still download the message so they do not learn that we have this message downloaded (or did at one point)
hostIsMalicious = true
messageHashesToDownloadList = append(messageHashesToDownloadList, receivedMessageHash)
continue
}
storedMessageExists, _, err := badgerDatabase.GetChatMessage(receivedMessageHash)
if (err != nil) { return false, err }
if (storedMessageExists == true){
// We already have the message, skip downloading it
continue
}
// Message does not exist, we must check if we want to download it
// This works differently for Hosting, Moderation, and Viewing
// For instance, if the user is downloading messages for viewing, they will not redownload profiles that they have already imported/deleted
shouldDownload, err := checkIfMessageShouldBeDownloaded(receivedMessageHash)
if (err != nil) { return false, err }
if (shouldDownload == true){
messageHashesToDownloadList = append(messageHashesToDownloadList, receivedMessageHash)
continue
}
}
if (len(messageHashesToDownloadList) == 0){
// Nothing to download, done with this host.
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
return true, nil
}
// Now we retrieve messages from host
maximumMessagesInResponse := serverResponse.MaximumMessagesInResponse_GetMessages
messageHashesSublistsList, err := helpers.SplitListIntoSublists(messageHashesToDownloadList, maximumMessagesInResponse)
if (err != nil) { return false, err }
numberOfDownloadedMessages := 0
for _, messageHashesSublist := range messageHashesSublistsList{
downloadSuccessful, messagesList, err := sendRequests.GetMessagesFromHost(connectionIdentifier, hostIdentityHash, networkType, messageHashesSublist, requestedInboxesRangeStart, requestedInboxesRangeEnd, requestedInboxesList)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Request has failed
// We will stop downloading from this host
break
}
for _, messageBytes := range messagesList{
messageIsWellFormed, err := chatMessageStorage.AddMessage(messageBytes)
if (err != nil) { return false, err }
if (messageIsWellFormed == false){
return false, errors.New("GetMessagesFromHost not verifying messages are well formed.")
}
}
numberOfDownloadedMessages += len(messagesList)
}
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
if (numberOfDownloadedMessages != 0){
numberOfMessagesDownloadedString := helpers.ConvertIntToString(numberOfDownloadedMessages)
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfMessagesDownloadedString + " messages from host.")
if (err != nil) { return false, err }
}
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadMessagesFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadIdentityReviewsFromHosts(
allowClearnet bool,
networkType byte,
identityTypeToRetrieve string,
rangeToRetrieveStart [16]byte,
rangeToRetrieveEnd [16]byte,
reviewedIdentityHashesToRetrieveList [][16]byte,
reviewersToRetrieveList [][16]byte,
checkIfReviewShouldBeDownloaded func([29]byte, []byte)(bool, error),
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadIdentityReviewsFromHosts called with invalid networkType: " + networkTypeString)
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReviewsFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[29]byte][]byte: Map of review info: ReviewHash -> ReviewedHash
// -[16]byte: Expected Identity Start Range of retrieved reviews
// -[16]byte: Expected Identity End range of retrieved reviews
// -[][16]byte: Expected retrieved identity hashes list
// -error
getRetrievedReviewsInfoMap := func()(bool, [21]byte, map[[29]byte][]byte, [16]byte, [16]byte, [][16]byte, error){
hostIsHostingIdentityType, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, identityTypeToRetrieve)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostIsHostingIdentityType == false){
// Host is not hosting any profiles/reviews of this identityType. Skip to next host
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
exists, identityTypeIdentitiesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, identityTypeToRetrieve + "IdentitiesQuantity")
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (exists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "IdentitiesQuantity")
}
identityTypeIdentitiesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeIdentitiesQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity)
}
getMaximumIdentitiesPerRequest := func()(int64, error){
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetIdentityReviewsInfo
if (len(reviewersToRetrieveList) != 0){
// We assume each reviewer has reviewed each identity and all it's profiles
profilesQuantityAttributeName := identityTypeToRetrieve + "ProfilesQuantity"
exists, identityTypeProfilesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, profilesQuantityAttributeName)
if (err != nil) { return 0, err }
if (exists == false){
return 0, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ProfilesQuantity")
}
identityTypeProfilesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeProfilesQuantity)
if (err != nil) { return 0, err }
estimatedNumberOfProfilesPerIdentity := identityTypeProfilesQuantityInt64 / identityTypeIdentitiesQuantityInt64
numberOfReviewersToRetrieve := int64(len(reviewersToRetrieveList))
estimatedReviewsPerIdentity := numberOfReviewersToRetrieve * (estimatedNumberOfProfilesPerIdentity + 1)
maximumIdentitiesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerIdentity
return maximumIdentitiesPerRequest, nil
}
reviewsQuantityAttributeName := identityTypeToRetrieve + "ReviewsQuantity"
exists, identityTypeReviewsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, reviewsQuantityAttributeName)
if (err != nil) { return 0, err }
if (exists == false){
return 0, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ReviewsQuantity")
}
identityTypeReviewsQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeReviewsQuantity)
if (err != nil) { return 0, err }
estimatedReviewsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReviewsQuantityInt64
maximumIdentitiesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerIdentity
return maximumIdentitiesPerRequest, nil
}
maximumIdentitiesPerRequest, err := getMaximumIdentitiesPerRequest()
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
// Our request will either be based on identity hash range or identity hashes list
// We will split up the request into either ranges or identity hashes
if (len(reviewedIdentityHashesToRetrieveList) != 0){
// We will split the request into identity hash lists
hostingAny, identitiesInHostRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, reviewedIdentityHashesToRetrieveList)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostingAny == false){
// Host is not hosting any of our desired identities. Skip host.
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
// Below step is just being overly cautious.
// The GetIdentityReviewsfromHosts function should request either a list of identities or a range. Not both.
hostingAny, identitiesInRequestRange, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identitiesInHostRangeList)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostingAny == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Requested review identity hashes list outside of request range")
}
sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInRequestRange, int(maximumIdentitiesPerRequest))
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds()
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][]byte, error){
retrievedReviewsInfoMap := make(map[[29]byte][]byte)
for _, identityHashesSublist := range sublistsToQueryList{
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetIdentityReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist, reviewersToRetrieveList)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop download from host.
err := logger.AddLogEntry("Network", "Failed to download identity reviews info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
retrievedReviewsInfoMap[reviewHash] = reviewedHash
}
}
return true, retrievedReviewsInfoMap, nil
}
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedReviewsInfoMap, minimumRange, maximumRange, identitiesInRequestRange, nil
}
// We will retrieve based on identity range, not identities list
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (anyIntersectionFound == false){
// Host is not hosting any reviews within the identites range that we are requesting
// Skip this host
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
estimatedNumIdentitiesInIntersectionRange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, identityTypeIdentitiesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumIdentitiesInIntersectionRange, maximumIdentitiesPerRequest)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][]byte, error){
retrievedReviewsInfoMap := make(map[[29]byte][]byte)
for _, subrangeObject := range subrangesToQueryList{
subrangeStart := subrangeObject.SubrangeStart
subrangeEnd := subrangeObject.SubrangeEnd
emptyList := make([][16]byte, 0)
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetIdentityReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, subrangeStart, subrangeEnd, emptyList, reviewersToRetrieveList)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop download.
err := logger.AddLogEntry("Network", "Failed to download identity reviews info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
retrievedReviewsInfoMap[reviewHash] = reviewedHash
}
}
return true, retrievedReviewsInfoMap, nil
}
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedReviewsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil, nil
}
hostDownloadSuccessful, connectionIdentifier, retrievedReviewsInfoMap, expectedRangeStart, expectedRangeEnd, expectedReviewedIdentityHashesList, err := getRetrievedReviewsInfoMap()
if (err != nil) { return false, err }
if (hostDownloadSuccessful == false){
// We did not download any reviews from this host. Skip them.
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
// Now we see if we have any of these reviews, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// This requires checking against content stored in our database
// A host could use this to learn information about what content we are storing
// We still download the reviews to prevent host from knowing that we have the reviewed content downloaded
// Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response
// Only after the full request is finished do we mark the host as malicious
// TODO: We shouldn't have to wait until the end of the request if we are only in Host/Moderator mode
// This is because we don't need to worry about leaking what content we have stored
reviewHashesToDownloadList := make([][29]byte, 0)
hostIsMalicious := false
for receivedReviewHash, receivedReviewedHash := range retrievedReviewsInfoMap{
storedReviewExists, storedReviewBytes, err := badgerDatabase.GetReview(receivedReviewHash)
if (err != nil) { return false, err }
checkIfHostIsOfferingInvalidReview := func()(bool, error){
if (storedReviewExists == true){
ableToRead, storedReviewHash, _, reviewNetworkType, storedReviewReviewerIdentity, _, storedReviewType, storedReviewReviewedHash, _, _, err := readReviews.ReadReviewAndHash(false, storedReviewBytes)
if (err != nil) { return false, err }
if (ableToRead == false){
return false, errors.New("Database corrupt: Contains invalid review.")
}
if (storedReviewHash != receivedReviewHash){
return false, errors.New("Database corrupt: Contains review with invalid entry key hash")
}
if (reviewNetworkType != networkType){
return true, nil
}
if (len(reviewersToRetrieveList) != 0){
isInList := slices.Contains(reviewersToRetrieveList, storedReviewReviewerIdentity)
if (isInList == false){
return true, nil
}
}
areEqual := bytes.Equal(storedReviewReviewedHash, receivedReviewedHash)
if (areEqual == false){
return true, nil
}
if (storedReviewType != "Identity" && storedReviewType != "Profile" && storedReviewType != "Attribute"){
return true, nil
}
}
// Now we check to see if we are aware of the reviewed content's metadata, and if it matches our request parameters
receivedReviewedType, err := helpers.GetReviewedTypeFromReviewedHash(receivedReviewedHash)
if (err != nil) { return false, err }
if (receivedReviewedType == "Profile"){
// We must verify that the profile's identity hash is expected
// We don't have to do this for identities, because we already checked that in the GetIdentityReviewsInfo function
if (len(receivedReviewedHash) != 28){
receivedReviewedHashHex := encoding.EncodeBytesToHexString(receivedReviewedHash)
return false, errors.New("GetReviewedTypeFromReviewedHash returning Profile for different length reviewedHash: " + receivedReviewedHashHex)
}
receivedReviewedProfileHash := [28]byte(receivedReviewedHash)
metadataIsKnown, _, profileNetworkType, reviewedProfileAuthor, _, profileIsDisabled, _, _, err := contentMetadata.GetProfileMetadata(receivedReviewedProfileHash)
if (err != nil) { return false, err }
if (metadataIsKnown == false){
return false, nil
}
if (profileIsDisabled == true){
// Disabled profiles cannot be reviewed. Host must be malicious.
return true, nil
}
if (profileNetworkType != networkType){
// Review is reviewing profile from different networkType
// Host should have checked this already. Host must be malicious.
return true, nil
}
if (expectedReviewedIdentityHashesList != nil){
identityIsInList := slices.Contains(expectedReviewedIdentityHashesList, reviewedProfileAuthor)
if (identityIsInList == false){
return true, nil
}
} else {
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, reviewedProfileAuthor)
if (err != nil) { return false, err }
if (identityIsWithinRange == false){
return true, nil
}
}
}
if (receivedReviewedType == "Attribute"){
if (len(receivedReviewedHash) != 27){
receivedReviewedHashHex := encoding.EncodeBytesToHexString(receivedReviewedHash)
return false, errors.New("GetReviewedTypeFromReviewedHash returning Attribute for different length reviewedHash: " + receivedReviewedHashHex)
}
receivedReviewedAttributeHash := [27]byte(receivedReviewedHash)
metadataExists, _, attributeAuthorIdentityHash, attributeNetworkType, _, err := profileStorage.GetProfileAttributeMetadata(receivedReviewedAttributeHash)
if (err != nil){ return false, err }
if (metadataExists == true){
if (expectedReviewedIdentityHashesList != nil){
identityIsInList := slices.Contains(expectedReviewedIdentityHashesList, attributeAuthorIdentityHash)
if (identityIsInList == false){
return true, nil
}
} else {
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, attributeAuthorIdentityHash)
if (err != nil) { return false, err }
if (identityIsWithinRange == false){
return true, nil
}
}
if (attributeNetworkType != networkType){
// Review is reviewing attribute from different networkType
// Host should have checked this already. Host must be malicious.
return true, nil
}
return false, nil
}
}
// Review is not invalid, as far as we know
return false, nil
}
hostIsOfferingInvalidReview, err := checkIfHostIsOfferingInvalidReview()
if (err != nil) { return false, err }
if (hostIsOfferingInvalidReview == true){
// Host is malicious, we must still download the review
hostIsMalicious = true
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
continue
}
if (storedReviewExists == true){
// We already have the review, skip downloading it
continue
}
// Review does not exist, we must check if we want to download it
shouldDownload, err := checkIfReviewShouldBeDownloaded(receivedReviewHash, receivedReviewedHash)
if (err != nil) { return false, err }
if (shouldDownload == true){
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
continue
}
}
if (len(reviewHashesToDownloadList) == 0){
// Nothing to download, we are done with this host.
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
return true, nil
}
// Now we download reviews
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetReviews
reviewHashesSublistsList, err := helpers.SplitListIntoSublists(reviewHashesToDownloadList, maximumReviewsInResponse)
if (err != nil) { return false, err }
numberOfDownloadedReviews := 0
for _, reviewHashesSublist := range reviewHashesSublistsList{
downloadSuccessful, reviewsList, err := sendRequests.GetReviewsFromHost(connectionIdentifier, hostIdentityHash, networkType, reviewHashesSublist, retrievedReviewsInfoMap, reviewersToRetrieveList)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Request has failed
// We will stop downloading from this host
break
}
if (hostIsMalicious == true){
// We don't add reviews from malicious hosts
continue
}
for _, reviewBytes := range reviewsList{
reviewIsWellFormed, err := reviewStorage.AddReview(reviewBytes)
if (err != nil) { return false, err }
if (reviewIsWellFormed == false){
return false, errors.New("GetReviewsFromHost not verifying reviews are well formed.")
}
}
numberOfDownloadedReviews += len(reviewsList)
}
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
if (numberOfDownloadedReviews != 0){
numberOfReviewsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReviews)
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReviewsDownloadedString + " identity reviews from host.")
if (err != nil) { return false, err }
}
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadReviewsFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadMessageReviewsFromHosts(
allowClearnet bool,
networkType byte,
rangeToRetrieveStart [10]byte,
rangeToRetrieveEnd [10]byte,
reviewedMessageHashesToRetrieveList [][26]byte,
reviewersToRetrieveList [][16]byte,
reviewedMessageInboxesMap map[[26]byte][10]byte, // Message hash -> Message inbox
checkIfReviewShouldBeDownloaded func([29]byte, [26]byte)(bool, error),
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadMessageReviewsFromHosts called with invalid networkType: " + networkTypeString)
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReviewsFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[29]byte][26]byte: Map of review info: ReviewHash -> ReviewedHash
// -[10]byte: Expected Inbox Start Range of retrieved reviews
// -[10]byte: Expected Inbox End range of retrieved reviews
// -error
getRetrievedReviewsInfoMap := func()(bool, [21]byte, map[[29]byte][26]byte, [10]byte, [10]byte, error){
hostIsHostingMessages, theirInboxRangeStart, theirInboxRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (hostIsHostingMessages == false){
// Host is not hosting messages. Nothing to download.
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity")
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (exists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessagesQuantity")
}
hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity)
}
exists, hostedMessageReviewsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessageReviewsQuantity")
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (exists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessageReviewsQuantity")
}
hostedMessageReviewsQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessageReviewsQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessageReviewsQuantity: " + hostedMessageReviewsQuantity)
}
getMaximumMessageHashesPerRequest := func()int{
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetMessageReviewsInfo
if (len(reviewersToRetrieveList) != 0){
// We assume each reviewer has 1 review per message
numberOfReviewersToRetrieve := len(reviewersToRetrieveList)
maximumMessageHashesPerRequest := maximumReviewsInResponse / numberOfReviewersToRetrieve
return maximumMessageHashesPerRequest
}
// We get the estimated number of reviews per message
estimatedReviewsPerMessage := hostedMessageReviewsQuantityInt64 / hostedMessagesQuantityInt64
maximumMessageHashesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerMessage
return int(maximumMessageHashesPerRequest)
}
maximumMessageHashesPerRequest := getMaximumMessageHashesPerRequest()
// We will retrieve reviews from either the reviewed hashes list, or request range
if (len(reviewedMessageHashesToRetrieveList) != 0){
// We will retrieve reviewed hashes list
messageHashesInTheirRangeList := make([][26]byte, 0)
for _, messageHash := range reviewedMessageHashesToRetrieveList{
messageInbox, exists := reviewedMessageInboxesMap[messageHash]
if (exists == false) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("reviewedMessageInboxesMap missing message hash.")
}
inboxIsWithinTheirRange, err := byteRange.CheckIfInboxIsWithinRange(theirInboxRangeStart, theirInboxRangeEnd, messageInbox)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (inboxIsWithinTheirRange == true){
messageHashesInTheirRangeList = append(messageHashesInTheirRangeList, messageHash)
}
}
sublistsToQueryList, err := helpers.SplitListIntoSublists(messageHashesInTheirRangeList, maximumMessageHashesPerRequest)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds()
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
//Outputs:
// -bool: Download successful
// -map[[29]byte][26]byte: Reviews info map (Review hash -> Reviewed message hash)
// -error
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][26]byte, error){
retrievedReviewsInfoMap := make(map[[29]byte][26]byte)
for _, messageHashesSublist := range sublistsToQueryList{
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetMessageReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, messageHashesSublist, reviewersToRetrieveList)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop retrieving from this host
err := logger.AddLogEntry("Network", "Failed to download message reviews info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
retrievedReviewsInfoMap[reviewHash] = reviewedHash
}
}
return true, retrievedReviewsInfoMap, nil
}
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
if (err != nil){
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
return true, connectionIdentifier, retrievedReviewsInfoMap, minimumRange, maximumRange, nil
}
// We will retrieve reviews based on message inbox range
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxRangeStart, theirInboxRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (anyIntersectionFound == false){
// Host is not hosting any reviews within the inboxes range that we are requesting
// Skip this host
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxRangeStart, theirInboxRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, int64(maximumMessageHashesPerRequest))
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
//Outputs:
// -bool: Download successful
// -map[[29]byte][26]byte: Downloaded reviews info map (Review Hash -> Reviewed Message Hash)
// -error
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][26]byte, error){
retrievedReviewsInfoMap := make(map[[29]byte][26]byte)
for _, subrangeObject := range subrangesToQueryList{
subrangeStart := subrangeObject.SubrangeStart
subrangeEnd := subrangeObject.SubrangeEnd
emptyList := make([][26]byte, 0)
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetMessageReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, emptyList, reviewersToRetrieveList)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, we are done with this host
err := logger.AddLogEntry("Network", "Failed to download message reviews info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
retrievedReviewsInfoMap[reviewHash] = reviewedHash
}
}
return true, retrievedReviewsInfoMap, nil
}
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
return true, connectionIdentifier, retrievedReviewsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil
}
hostDownloadSuccessful, connectionIdentifier, retrievedReviewsInfoMap, expectedRangeStart, expectedRangeEnd, err := getRetrievedReviewsInfoMap()
if (err != nil) { return false, err }
if (hostDownloadSuccessful == false){
// We did not download any reviews from this host. Skip them.
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
// Now we see if we have any of these reviews, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// This requires checking against content stored in our database
// A host could use this to learn information about what messages we are storing
// We still download the reviews to prevent host from knowing that we have the reviewed messages downloaded
// Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response
// Only after the full request is finished do we mark the host as malicious
// TODO: We shouldn't have to wait until the end of the request if we are only in Host mode
// This is because if we we don't need to worry about leaking what messages we have stored
reviewHashesToDownloadList := make([][29]byte, 0)
hostIsMalicious := false
for receivedReviewHash, receivedReviewedHash := range retrievedReviewsInfoMap{
storedReviewExists, storedReviewBytes, err := badgerDatabase.GetReview(receivedReviewHash)
if (err != nil) { return false, err }
checkIfHostIsOfferingInvalidReview := func()(bool, error){
if (storedReviewExists == true){
ableToRead, storedReviewHash, _, reviewNetworkType, storedReviewReviewerIdentity, _, storedReviewType, storedReviewReviewedHash, _, _, err := readReviews.ReadReviewAndHash(false, storedReviewBytes)
if (err != nil) { return false, err }
if (ableToRead == false){
return false, errors.New("Database corrupt: Contains invalid review.")
}
if (storedReviewHash != receivedReviewHash){
return false, errors.New("Database corrupt: Contains review with invalid entry key hash")
}
if (reviewNetworkType != networkType){
return true, nil
}
if (len(reviewersToRetrieveList) != 0){
isInList := slices.Contains(reviewersToRetrieveList, storedReviewReviewerIdentity)
if (isInList == false){
return true, nil
}
}
areEqual := bytes.Equal(storedReviewReviewedHash, receivedReviewedHash[:])
if (areEqual == false){
return true, nil
}
if (storedReviewType != "Message"){
return true, nil
}
}
// Now we check to see if we are aware of the reviewed message's metadata, and if it matches our request parameters
metadataIsKnown, _, messageNetworkType, _, reviewedMessageInbox, _, err := contentMetadata.GetMessageMetadata(receivedReviewedHash)
if (err != nil) { return false, err }
if (metadataIsKnown == false){
return false, nil
}
if (messageNetworkType != networkType){
// The review is reviewing a message from a different networkType
// The host should have checked for this before serving us the review
// Thus, the host must be malicious
return true, nil
}
messageInboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(expectedRangeStart, expectedRangeEnd, reviewedMessageInbox)
if (err != nil) { return false, err }
if (messageInboxIsWithinRange == false){
return false, nil
}
return false, nil
}
hostIsOfferingInvalidReview, err := checkIfHostIsOfferingInvalidReview()
if (err != nil) { return false, err }
if (hostIsOfferingInvalidReview == true){
// Host is malicious, we must still download the review
hostIsMalicious = true
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
continue
}
if (storedReviewExists == true){
// We already have the review, skip downloading it
continue
}
// Review does not exist, we must check if we want to download it
shouldDownload, err := checkIfReviewShouldBeDownloaded(receivedReviewHash, receivedReviewedHash)
if (err != nil) { return false, err }
if (shouldDownload == true){
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
continue
}
}
if (len(reviewHashesToDownloadList) == 0){
// Nothing to download, done with this host.
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
return true, nil
}
// Now we download reviews
// We convert the [26]byte values to []byte values to pass to GetReviewsFromHost
reviewsInfoMapForRequest := make(map[[29]byte][]byte)
for reviewHash, reviewedMessageHash := range retrievedReviewsInfoMap{
reviewsInfoMapForRequest[reviewHash] = reviewedMessageHash[:]
}
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetReviews
reviewHashesSublistsList, err := helpers.SplitListIntoSublists(reviewHashesToDownloadList, maximumReviewsInResponse)
if (err != nil) { return false, err }
numberOfDownloadedReviews := 0
for _, reviewHashesSublist := range reviewHashesSublistsList{
downloadSuccessful, reviewsList, err := sendRequests.GetReviewsFromHost(connectionIdentifier, hostIdentityHash, networkType, reviewHashesSublist, reviewsInfoMapForRequest, reviewersToRetrieveList)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Request has failed
// We will stop downloading from this host
break
}
if (hostIsMalicious == true){
// We don't add reviews from malicious hosts
continue
}
for _, reviewBytes := range reviewsList{
reviewIsWellFormed, err := reviewStorage.AddReview(reviewBytes)
if (err != nil) { return false, err }
if (reviewIsWellFormed == false){
return false, errors.New("GetReviewsFromHost not verifying reviews are well formed.")
}
}
numberOfDownloadedReviews += len(reviewsList)
}
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
if (numberOfDownloadedReviews != 0){
numberOfReviewsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReviews)
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReviewsDownloadedString + " reviews from host.")
if (err != nil) { return false, err }
}
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadReviewsFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadIdentityReportsFromHosts(
allowClearnet bool,
networkType byte,
identityTypeToRetrieve string,
rangeToRetrieveStart [16]byte,
rangeToRetrieveEnd [16]byte,
reportedIdentityHashesToRetrieveList [][16]byte,
checkIfReportShouldBeDownloaded func([30]byte, []byte)(bool, error),
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadIdentityReportsFromHosts called with invalid networkType: " + networkTypeString)
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReportsFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[30]byte][]byte: Map of report info: ReportHash -> ReportedHash
// -[16]byte: Expected Identity Start Range of retrieved reports
// -[16]byte: Expected Identity End range of retrieved reports
// -[][16]byte: Expected retrieved identity hashes list
// -error
getRetrievedReportsInfoMap := func()(bool, [21]byte, map[[30]byte][]byte, [16]byte, [16]byte, [][16]byte, error){
hostIsHostingIdentityType, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, identityTypeToRetrieve)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostIsHostingIdentityType == false){
// Host is not hosting any profiles/reports of this identityType. Skip to next host
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
identitiesQuantityAttributeName := identityTypeToRetrieve + "IdentitiesQuantity"
exists, identityTypeIdentitiesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, identitiesQuantityAttributeName)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (exists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + identitiesQuantityAttributeName)
}
identityTypeIdentitiesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeIdentitiesQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains Host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity)
}
reportsQuantityAttributeName := identityTypeToRetrieve + "ReportsQuantity"
exists, identityTypeReportsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, reportsQuantityAttributeName)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (exists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + reportsQuantityAttributeName)
}
identityTypeReportsQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeReportsQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid " + reportsQuantityAttributeName + ": " + identityTypeReportsQuantity)
}
//TODO: Deal with divide by 0 case here and on all other functions
estimatedReportsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReportsQuantityInt64
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetIdentityReportsInfo
maximumIdentitiesPerRequest := int64(maximumReportsInResponse) / estimatedReportsPerIdentity
// Our request will either be based on identity hash range or identity hashes list
// We will split up the request into either ranges or identity hash lists
if (len(reportedIdentityHashesToRetrieveList) != 0){
// We will split the request into identity hash lists
hostingAny, identitiesInHostRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, reportedIdentityHashesToRetrieveList)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostingAny == false){
// Host is not hosting any of our desired identities. Skip host.
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
// Below step is just being overly cautious.
// The GetMessageReportsfromHosts function should request either a list of identities or a range. Not both.
hostingAny, identitiesInRequestRange, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identitiesInHostRangeList)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostingAny == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Requested report identity hashes list outside of request range")
}
sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInRequestRange, int(maximumIdentitiesPerRequest))
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds()
//Outputs:
// -bool: Download successful
// -map[[30]byte][]byte: Downloaded reports info map (Report Hash -> Reported Hash)
// -error
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][]byte, error){
retrievedReportsInfoMap := make(map[[30]byte][]byte)
for _, identityHashesSublist := range sublistsToQueryList{
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetIdentityReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop downloading from this host
err := logger.AddLogEntry("Network", "Failed to download identity reports info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
retrievedReportsInfoMap[reportHash] = reportedHash
}
}
return true, retrievedReportsInfoMap, nil
}
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedReportsInfoMap, minimumRange, maximumRange, identitiesInRequestRange, nil
}
// We will retrieve based on identity range, not identities list
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (anyIntersectionFound == false){
// Host is not hosting any reports within the identites range that we are requesting
// Skip this host
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
estimatedNumIdentitiesInIntersectionRange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, identityTypeIdentitiesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumIdentitiesInIntersectionRange, maximumIdentitiesPerRequest)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][]byte, error){
retrievedReportsInfoMap := make(map[[30]byte][]byte)
for _, subrangeObject := range subrangesToQueryList{
subrangeStart := subrangeObject.SubrangeStart
subrangeEnd := subrangeObject.SubrangeEnd
emptyList := make([][16]byte, 0)
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetIdentityReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, subrangeStart, subrangeEnd, emptyList)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, close connection to host.
err := logger.AddLogEntry("Network", "Failed to download identity reports info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
retrievedReportsInfoMap[reportHash] = reportedHash
}
}
return true, retrievedReportsInfoMap, nil
}
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
}
return true, connectionIdentifier, retrievedReportsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil, nil
}
hostDownloadSuccessful, connectionIdentifier, retrievedReportsInfoMap, expectedRangeStart, expectedRangeEnd, expectedReportedIdentityHashesList, err := getRetrievedReportsInfoMap()
if (err != nil) { return false, err }
if (hostDownloadSuccessful == false){
// We did not download any reports from this host. Skip them.
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
// Now we see if we have any of these reports, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// We still download the reports to prevent host from knowing that we have those reports downloaded
// We then mark the host as malicious
reportHashesToDownloadList := make([][30]byte, 0)
hostIsMalicious := false
for receivedReportHash, receivedReportedHash := range retrievedReportsInfoMap{
//TODO: Ability to disable this step if only 1 of Host/Moderator mode is enabled
storedReportExists, storedReportBytes, err := badgerDatabase.GetReport(receivedReportHash)
if (err != nil) { return false, err }
checkIfHostIsOfferingInvalidReport := func()(bool, error){
if (storedReportExists == true){
ableToRead, storedReportHash, _, reportNetworkType, _, storedReportType, storedReportReportedHash, _, err := readReports.ReadReportAndHash(false, storedReportBytes)
if (err != nil) { return false, err }
if (ableToRead == false){
return false, errors.New("Database corrupt: Contains invalid report")
}
if (storedReportHash != receivedReportHash){
return false, errors.New("Database corrupt: Contains report with invalid entry key hash")
}
if (reportNetworkType != networkType){
return true, nil
}
areEqual := bytes.Equal(storedReportReportedHash, receivedReportedHash)
if (areEqual == false){
return true, nil
}
if (storedReportType != "Identity" && storedReportType != "Profile" && storedReportType != "Attribute"){
return true, nil
}
}
// Now we see if we have the reviewed profile metadata
reportedType, err := helpers.GetReportedTypeFromReportedHash(receivedReportedHash)
if (err != nil){ return false, err }
if (reportedType == "Profile"){
if (len(receivedReportedHash) != 28){
reportedHashHex := encoding.EncodeBytesToHexString(receivedReportedHash)
return false, errors.New("GetReportedTypeFromReportedHash returning Profile for different length reportedHash: " + reportedHashHex)
}
receivedReportedProfileHash := [28]byte(receivedReportedHash)
metadataExists, _, profileNetworkType, profileAuthor, _, profileIsDisabled, _, _, err := contentMetadata.GetProfileMetadata(receivedReportedProfileHash)
if (err != nil) { return false, err }
if (metadataExists == false){
return false, nil
}
if (profileNetworkType != networkType){
// Reported profile belongs to a different networkType than the report
// The host should have checked for this before serving this report
// Host must be malicious
return true, nil
}
if (profileIsDisabled == true){
// Disabled profiles cannot be reported
// Host must be malicious
return true, nil
}
if (expectedReportedIdentityHashesList != nil){
identityIsInList := slices.Contains(expectedReportedIdentityHashesList, profileAuthor)
if (identityIsInList == false){
return true, nil
}
} else {
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, profileAuthor)
if (err != nil) { return false, err }
if (identityIsWithinRange == false){
return true, nil
}
}
} else if (reportedType == "Attribute"){
if (len(receivedReportedHash) != 27){
reportedHashHex := encoding.EncodeBytesToHexString(receivedReportedHash)
return false, errors.New("GetReportedTypeFromReportedHash returning Attribute for different length reportedHash: " + reportedHashHex)
}
receivedReportedAttributeHash := [27]byte(receivedReportedHash)
metadataExists, _, attributeAuthorIdentityHash, attributeNetworkType, _, err := profileStorage.GetProfileAttributeMetadata(receivedReportedAttributeHash)
if (err != nil){ return false, err }
if (metadataExists == true){
if (expectedReportedIdentityHashesList != nil){
identityIsInList := slices.Contains(expectedReportedIdentityHashesList, attributeAuthorIdentityHash)
if (identityIsInList == false){
return true, nil
}
} else {
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, attributeAuthorIdentityHash)
if (err != nil) { return false, err }
if (identityIsWithinRange == false){
return true, nil
}
}
if (attributeNetworkType != networkType){
// Report is reporting attribute from different networkType
// Host should have checked this already. Host must be malicious.
return true, nil
}
return false, nil
}
}
// Host is not offering invalid report (as far as we know)
return false, nil
}
hostIsOfferingInvalidReport, err := checkIfHostIsOfferingInvalidReport()
if (err != nil) { return false, err }
if (hostIsOfferingInvalidReport == true){
// Host is malicious, we must still download the report
hostIsMalicious = true
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
continue
}
if (storedReportExists == true){
// We already have the report, skip downloading it
continue
}
// Report does not exist, we must check if we want to download it
// This works differently for Hosting and Moderation
shouldDownload, err := checkIfReportShouldBeDownloaded(receivedReportHash, receivedReportedHash)
if (err != nil) { return false, err }
if (shouldDownload == true){
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
continue
}
}
if (len(reportHashesToDownloadList) == 0){
// Nothing to download, done with this host.
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
return true, nil
}
// Now we download reports
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetReports
reportHashesSublistsList, err := helpers.SplitListIntoSublists(reportHashesToDownloadList, maximumReportsInResponse)
if (err != nil) { return false, err }
numberOfDownloadedReports := 0
for _, reportHashesSublist := range reportHashesSublistsList{
downloadSuccessful, reportsList, err := sendRequests.GetReportsFromHost(connectionIdentifier, hostIdentityHash, networkType, reportHashesSublist, retrievedReportsInfoMap)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Request has failed
// We will stop downloading from this host
break
}
for _, reportBytes := range reportsList{
reportIsWellFormed, err := reportStorage.AddReport(reportBytes)
if (err != nil) { return false, err }
if (reportIsWellFormed == false){
return false, errors.New("GetReportsFromHost not verifying reports are well formed.")
}
}
numberOfDownloadedReports += len(reportsList)
}
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
if (numberOfDownloadedReports != 0){
numberOfReportsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReports)
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReportsDownloadedString + " identity reports from host.")
if (err != nil) { return false, err }
}
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadReportsFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadMessageReportsFromHosts(
allowClearnet bool,
networkType byte,
rangeToRetrieveStart [10]byte,
rangeToRetrieveEnd [10]byte,
reportedMessageHashesToRetrieveList [][26]byte,
reportedMessagesInboxMap map[[26]byte][10]byte, // Message hash -> Message inbox
checkIfReportShouldBeDownloaded func([30]byte, [26]byte)(bool, error),
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadMessageReportsFromHosts called with invalid networkType: " + networkTypeString)
}
//TODO: Verify inputs
//Outputs:
// -bool: Download successful
// -error
downloadReportsFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
// Outputs:
// -bool: Successfully downloaded from host
// -[21]byte: Connection identifier
// -map[[30]byte][26]byte: Map of report info: ReportHash-> Reported message Hash
// -[10]byte: Expected Inbox Start Range of retrieved reports
// -[10]byte: Expected Inbox End range of retrieved reports
// -error
getRetrievedReportsInfoMap := func()(bool, [21]byte, map[[30]byte][26]byte, [10]byte, [10]byte, error){
hostIsHostingMessages, theirInboxHostRangeStart, theirInboxHostRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (hostIsHostingMessages == false){
// Host is not hosting messages. Nothing to download.
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity")
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (exists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessagesQuantity")
}
hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity)
}
exists, hostedMessageReportsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessageReportsQuantity")
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (exists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessageReportsQuantity")
}
hostedMessageReportsQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessageReportsQuantity)
if (err != nil) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Databse corrupt: Contains host profile with invalid MessageReportsQuantity: " + hostedMessageReportsQuantity)
}
if (hostedMessageReportsQuantityInt64 == 0){
// Host is not hosting any message reports. Nothing to download.
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetMessageReportsInfo
estimatedReportsPerMessage := hostedMessageReportsQuantityInt64 / hostedMessagesQuantityInt64
maximumMessageHashesPerRequest := int64(maximumReportsInResponse) / estimatedReportsPerMessage
// We will retrieve reports from either the reported hashes list, or request range
if (len(reportedMessageHashesToRetrieveList) != 0){
// We will retrieve reported message hashes list
messageHashesInTheirRangeList := make([][26]byte, 0)
for _, messageHash := range reportedMessageHashesToRetrieveList{
messageInbox, exists := reportedMessagesInboxMap[messageHash]
if (exists == false) {
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("ReportedMessagesInboxMap missing message hash.")
}
inboxIsWithinTheirRange, err := byteRange.CheckIfInboxIsWithinRange(theirInboxHostRangeStart, theirInboxHostRangeEnd, messageInbox)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (inboxIsWithinTheirRange == true){
messageHashesInTheirRangeList = append(messageHashesInTheirRangeList, messageHash)
}
}
sublistsToQueryList, err := helpers.SplitListIntoSublists(messageHashesInTheirRangeList, int(maximumMessageHashesPerRequest))
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds()
//Outputs:
// -bool: Download successful
// -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported Message Hash)
// -error
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][26]byte, error){
retrievedReportsInfoMap := make(map[[30]byte][26]byte)
for _, messageHashesSublist := range sublistsToQueryList{
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetMessageReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, messageHashesSublist)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, end connection to host
err := logger.AddLogEntry("Network", "Failed to download message reports info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
retrievedReportsInfoMap[reportHash] = reportedHash
}
}
return true, retrievedReportsInfoMap, nil
}
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
return true, connectionIdentifier, retrievedReportsInfoMap, minimumRange, maximumRange, nil
}
// We will retrieve reports based on message inbox range
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxHostRangeStart, theirInboxHostRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (anyIntersectionFound == false){
// Host is not hosting any reports within the inboxes range that we are requesting
// Skip this host
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxHostRangeStart, theirInboxHostRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, maximumMessageHashesPerRequest)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
if (hostProfileExists == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
if (connectionEstablished == false){
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
//Outputs:
// -bool: Download successful
// -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported message hash)
// -error
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][26]byte, error){
retrievedReportsInfoMap := make(map[[30]byte][26]byte)
for _, subrangeObject := range subrangesToQueryList{
subrangeStart := subrangeObject.SubrangeStart
subrangeEnd := subrangeObject.SubrangeEnd
emptyList := make([][26]byte, 0)
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetMessageReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, emptyList)
if (err != nil) { return false, nil, err }
if (downloadSuccessful == false){
// Failed to download this sublist.
// Could be that host went offline permanently or just this connection failed.
// Either way, stop downloading from host
err := logger.AddLogEntry("Network", "Failed to download message reports info from host.")
if (err != nil) { return false, nil, err }
return false, nil, nil
}
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
retrievedReportsInfoMap[reportHash] = reportedHash
}
}
return true, retrievedReportsInfoMap, nil
}
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
if (err != nil) {
errB := peerClient.CloseConnection(connectionIdentifier)
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
}
if (downloadSuccessful == false){
err := peerClient.CloseConnection(connectionIdentifier)
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
}
return true, connectionIdentifier, retrievedReportsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil
}
hostDownloadSuccessful, connectionIdentifier, retrievedReportsInfoMap, expectedRangeStart, expectedRangeEnd, err := getRetrievedReportsInfoMap()
if (err != nil) { return false, err }
if (hostDownloadSuccessful == false){
// We did not download any reports from this host. Skip them.
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
// Now we see if we have any of these reports, and if they do not fulfill the requests parameters
// If they do not fulfill the request, we know host is malicious
// We still download the reports to prevent host from knowing that we have those reports downloaded
// We then mark the host as malicious
reportHashesToDownloadList := make([][30]byte, 0)
hostIsMalicious := false
for receivedReportHash, receivedReportedHash := range retrievedReportsInfoMap{
//TODO: Disable this step if only 1 of Host/Moderator mode is enabled
storedReportExists, storedReportBytes, err := badgerDatabase.GetReport(receivedReportHash)
if (err != nil) { return false, err }
checkIfHostIsOfferingInvalidReport := func()(bool, error){
if (storedReportExists == true){
ableToRead, storedReportHash, _, reportNetworkType, _, storedReportType, storedReportReportedHash, _, err := readReports.ReadReportAndHash(false, storedReportBytes)
if (err != nil) { return false, err }
if (ableToRead == false){
return false, errors.New("Database corrupt: Contains invalid report")
}
if (storedReportHash != receivedReportHash){
return false, errors.New("Database corrupt: Contains report with invalid entry key hash")
}
if (reportNetworkType != networkType){
return true, nil
}
areEqual := bytes.Equal(storedReportReportedHash, receivedReportedHash[:])
if (areEqual == false){
return true, nil
}
if (storedReportType != "Message"){
return true, nil
}
}
// Now we see if we have the reviewed message metadata
metadataExists, _, messageNetworkType, _, reportedMessageInbox, _, err := contentMetadata.GetMessageMetadata(receivedReportedHash)
if (err != nil) { return false, err }
if (metadataExists == false){
return false, nil
}
if (messageNetworkType != networkType){
// Report is reporting a message on a different networkType than the report
// The host should have checked for this already
// Host must be malicious
return true, nil
}
inboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(expectedRangeStart, expectedRangeEnd, reportedMessageInbox)
if (err != nil) { return false, err }
if (inboxIsWithinRange == false){
return true, nil
}
return false, nil
}
hostIsOfferingInvalidReport, err := checkIfHostIsOfferingInvalidReport()
if (err != nil) { return false, err }
if (hostIsOfferingInvalidReport == true){
// Host is malicious, we must still download the report
hostIsMalicious = true
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
continue
}
if (storedReportExists == true){
// We already have the report, skip downloading it
continue
}
// Report does not exist, we must check if we want to download it
// This works differently for Hosting and Moderation
// For instance, if the user is downloading reports for moderation, they will not redownload reports for messages that they have already reviewed
shouldDownload, err := checkIfReportShouldBeDownloaded(receivedReportHash, receivedReportedHash)
if (err != nil) { return false, err }
if (shouldDownload == true){
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
continue
}
}
if (len(reportHashesToDownloadList) == 0){
// Nothing to download, done with this host.
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
return true, nil
}
// Now we download reports
// We convert the [26]byte value to []byte to supply to GetReportsFromHost
reportsInfoMapForRequest := make(map[[30]byte][]byte)
for reportHash, reportedMessageHash := range retrievedReportsInfoMap{
reportsInfoMapForRequest[reportHash] = reportedMessageHash[:]
}
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetReports
reportHashesSublistsList, err := helpers.SplitListIntoSublists(reportHashesToDownloadList, maximumReportsInResponse)
if (err != nil) { return false, err }
numberOfDownloadedReports := 0
for _, reportHashesSublist := range reportHashesSublistsList{
downloadSuccessful, reportsList, err := sendRequests.GetReportsFromHost(connectionIdentifier, hostIdentityHash, networkType, reportHashesSublist, reportsInfoMapForRequest)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// Request has failed
// We will stop downloading from this host
break
}
for _, reportBytes := range reportsList{
reportIsWellFormed, err := reportStorage.AddReport(reportBytes)
if (err != nil) { return false, err }
if (reportIsWellFormed == false){
return false, errors.New("GetReportsFromHost not verifying reports are well formed.")
}
}
numberOfDownloadedReports += len(reportsList)
}
if (hostIsMalicious == true){
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
if (err != nil) { return false, err }
}
if (numberOfDownloadedReports != 0){
numberOfReportsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReports)
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReportsDownloadedString + " reports from host.")
if (err != nil) { return false, err }
}
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadReportsFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
//Inputs:
// -bool: Allow clearnet
// -[]string: List of identity hashes to retrieve viewable consensus statuses for
// -map[[28]byte][16]byte: Map of Profile Hash -> Profile Identity Hash
// Outputs:
// -error
func DownloadViewableStatusesFromHosts(
allowClearnet bool,
networkType byte,
knownOrUnknown string,
identityHashesToRetrieveList [][16]byte,
profileHashesToRetrieveMap map[[28]byte][16]byte,
numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadViewableStatusesFromHosts called with invalid networkType: " + networkTypeString)
}
if (knownOrUnknown != "Known" && knownOrUnknown != "Unknown"){
return errors.New("DownloadViewableStatusesFromHosts called with invalid knownOrUnknown: " + knownOrUnknown)
}
//Outputs:
// -bool: Downloaded any statuses
// -error
downloadViewableStatusesFromHost := func(hostIdentityHash [16]byte)(bool, error){
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (exists == false){
// Host profile was deleted, skip host
return false, nil
}
hostIsHostingMateProfiles, hostMateRangeStart, hostMateRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Mate")
if (err != nil) { return false, err }
hostIsHostingHostProfiles, _, _, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Host")
if (err != nil) { return false, err }
hostIsHostingModeratorProfiles, _, _, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Moderator")
if (err != nil) { return false, err }
checkIfHostIsHostingIdentityHash := func(identityHash [16]byte)(bool, error){
identityType, err := identity.GetIdentityTypeFromIdentityHash(identityHash)
if (err != nil) { return false, err }
if (identityType == "Mate"){
if (hostIsHostingMateProfiles == false){
return false, nil
}
isWithinTheirRange, err := byteRange.CheckIfIdentityHashIsWithinRange(hostMateRangeStart, hostMateRangeEnd, identityHash)
if (err != nil) { return false, err }
if (isWithinTheirRange == false){
return false, nil
}
return true, nil
}
if (identityType == "Host"){
if (hostIsHostingHostProfiles == false){
return false, nil
}
return true, nil
}
// identityType == "Moderator"
if (hostIsHostingModeratorProfiles == false){
return false, nil
}
return true, nil
}
relevantIdentityHashesToRetrieveList := make([][16]byte, 0)
for _, identityHash := range identityHashesToRetrieveList{
isHostingIdentity, err := checkIfHostIsHostingIdentityHash(identityHash)
if (err != nil) { return false, err }
if (isHostingIdentity == false){
continue
}
statusIsKnown, _, queriedHostsList, err := trustedViewableStatus.GetTrustedIdentityIsViewableStatus(identityHash, networkType)
if (err != nil) { return false, err }
if (knownOrUnknown == "Known"){
// We only want to retrieve known statuses. We dont need to avoid already queried hosts.
if (statusIsKnown == false){
continue
}
} else {
// We only want to retrieve unknown statuses
if (statusIsKnown == true){
continue
}
// We must avoid hosts we have already queried
alreadyQueried := slices.Contains(queriedHostsList, hostIdentityHash)
if (alreadyQueried == true){
continue
}
}
relevantIdentityHashesToRetrieveList = append(relevantIdentityHashesToRetrieveList, identityHash)
}
relevantProfileHashesToRetrieveList := make([][28]byte, 0)
for profileHash, profileIdentityHash := range profileHashesToRetrieveMap{
isHostingIdentity, err := checkIfHostIsHostingIdentityHash(profileIdentityHash)
if (err != nil) { return false, err }
if (isHostingIdentity == false){
continue
}
statusIsKnown, _, queriedHostsList, err := trustedViewableStatus.GetTrustedProfileIsViewableStatus(profileHash)
if (err != nil) { return false, err }
if (knownOrUnknown == "Known"){
// We only want to retrieve known statuses. We dont need to avoid already queried hosts.
if (statusIsKnown == false){
continue
}
} else {
// We only want to retrieve unknown statuses
if (statusIsKnown == true){
continue
}
// We must avoid hosts we have already queried
alreadyQueried := slices.Contains(queriedHostsList, hostIdentityHash)
if (alreadyQueried == true){
continue
}
}
relevantProfileHashesToRetrieveList = append(relevantProfileHashesToRetrieveList, profileHash)
}
if (len(relevantIdentityHashesToRetrieveList) == 0 && len(relevantProfileHashesToRetrieveList) == 0){
// Nothing to retrieve from this host. Skip them.
return false, nil
}
// Now we must split up our request to not exceed maximum size
// The requests/responses for identity hashes will always be smaller than profile hashes, for the same number of identities/profiles
// This is because profile hashes are larger than identity hashes
// We will split request so that we can fit the maximum amount of identity/profile hashes in all of the requests
numberOfIdentityHashesToRetrieve := len(relevantIdentityHashesToRetrieveList)
numberOfProfileHashesToRetrieve := len(relevantProfileHashesToRetrieveList)
maximumIdentityHashesInResponse := serverResponse.MaximumIdentitiesInResponse_GetIdentityViewableStatuses
maximumProfileHashesInResponse := serverResponse.MaximumProfilesInResponse_GetProfileViewableStatuses
numberOfRequestsNeededForIdentityHashes := float64(numberOfIdentityHashesToRetrieve)/float64(maximumIdentityHashesInResponse)
numberOfRequestsNeededForProfileHashes := float64(numberOfProfileHashesToRetrieve)/float64(maximumProfileHashesInResponse)
totalNumberOfRequestsNeeded, err := helpers.CeilFloat64ToInt64(numberOfRequestsNeededForIdentityHashes + numberOfRequestsNeededForProfileHashes)
if (err != nil) { return false, err }
numberOfIdentityHashesPerRequest := int64(numberOfIdentityHashesToRetrieve)/totalNumberOfRequestsNeeded
numberOfProfileHashesPerRequest := int64(numberOfProfileHashesToRetrieve)/totalNumberOfRequestsNeeded
identityHashesSublistsList, err := helpers.SplitListIntoSublists(relevantIdentityHashesToRetrieveList, int(numberOfIdentityHashesPerRequest))
if (err != nil) { return false, err }
profileHashesSublistsList, err := helpers.SplitListIntoSublists(relevantProfileHashesToRetrieveList, int(numberOfProfileHashesPerRequest))
if (err != nil) { return false, err }
identityHashesSublistsListLength := len(identityHashesSublistsList)
profileHashesSublistsListLength := len(profileHashesSublistsList)
getLongerListLength := func()int{
if (identityHashesSublistsListLength > profileHashesSublistsListLength){
return identityHashesSublistsListLength
}
return profileHashesSublistsListLength
}
longerListLength := getLongerListLength()
totalDownloadedStatuses := 0
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, err }
if (hostProfileExists == false){
return false, nil
}
if (connectionEstablished == false){
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
for i := 0; i < longerListLength; i++ {
getIdentityHashesSublist := func()[][16]byte{
if (i < identityHashesSublistsListLength){
return identityHashesSublistsList[i]
}
emptyList := make([][16]byte, 0)
return emptyList
}
getProfileHashesSublist := func()[][28]byte{
if (i < profileHashesSublistsListLength){
return profileHashesSublistsList[i]
}
emptyList := make([][28]byte, 0)
return emptyList
}
identityHashesSublist := getIdentityHashesSublist()
profileHashesSublist := getProfileHashesSublist()
downloadSuccessful, identityHashStatusesMap, profileHashStatusesMap, err := sendRequests.GetViewableStatusesFromHost(connectionIdentifier, hostIdentityHash, networkType, identityHashesSublist, profileHashesSublist)
if (err != nil) { return false, err }
if (downloadSuccessful == false){
// This host has failed to respond. We will stop downloading from them.
return false, nil
}
for identityHash, viewableStatus := range identityHashStatusesMap{
err := trustedViewableStatus.AddTrustedIdentityIsViewableStatus(identityHash, hostIdentityHash, networkType, viewableStatus)
if (err != nil) { return false, err }
totalDownloadedStatuses += 1
}
for profileHash, viewableStatus := range profileHashStatusesMap{
err := trustedViewableStatus.AddTrustedProfileIsViewableStatus(profileHash, hostIdentityHash, viewableStatus)
if (err != nil) { return false, err }
profileIdentityHash, exists := profileHashesToRetrieveMap[profileHash]
if (exists == false){
return false, errors.New("profileHashesToRetrieveMap missing profileHash")
}
if (viewableStatus == true){
// The host will only indicate a viewable profile if they know that the identity is not banned
err := trustedViewableStatus.AddTrustedIdentityIsViewableStatus(profileIdentityHash, hostIdentityHash, networkType, true)
if (err != nil) { return false, err }
}
totalDownloadedStatuses += 1
}
}
totalDownloadedStatusesString := helpers.ConvertIntToString(totalDownloadedStatuses)
err = logger.AddLogEntry("Network", "Successfully downloaded " + totalDownloadedStatusesString + " viewable statuses from host.")
if (err != nil) { return false, err }
return true, nil
}
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil) { return err }
queriedHosts := 0
for _, hostIdentityHash := range eligibleHostsList{
successfulDownload, err := downloadViewableStatusesFromHost(hostIdentityHash)
if (err != nil) { return err }
if (successfulDownload == true){
queriedHosts += 1
}
if (queriedHosts >= numberOfHostsToQuery){
return nil
}
}
return nil
}
func DownloadModeratorAddressDepositsFromHosts(allowClearnet bool, networkType byte, cryptocurrencyName string, knownOrUnknown string, numberOfHostsToQuery int)error{
isValid := helpers.VerifyNetworkType(networkType)
if (isValid == false){
networkTypeString := helpers.ConvertByteToString(networkType)
return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid networkType: " + networkTypeString)
}
if (cryptocurrencyName != "Ethereum" && cryptocurrencyName != "Cardano"){
return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid cryptocurrencyName: " + cryptocurrencyName)
}
if (knownOrUnknown != "Unknown" && knownOrUnknown != "Known"){
return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid knownOrUnknown: " + knownOrUnknown)
}
//TODO: Check if local Ethereum/Cardano node is being used (either here or in function that calls this function)
//TODO: Check to see if less than 3 blockchain hosts exist, in which case, relax the required rule of 3 needed
// We will first see which identity deposits are known/unknown
//Outputs:
// -bool: Downloaded any deposits
// -error
downloadDepositsFromHost := func(hostIdentityHash [16]byte)(bool, error){
profileExists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
if (err != nil) { return false, err }
if (profileExists == false){
return false, nil
}
exists, isHostingBlockchain, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "Hosting" + cryptocurrencyName + "Blockchain")
if (err != nil) { return false, err }
if (exists == false || isHostingBlockchain != "Yes"){
return false, nil
}
localBlockchainNodeEnabled, addressesToUpdateList, err := moderatorScores.GetModeratorCryptoDepositAddressesList(cryptocurrencyName, knownOrUnknown)
if (err != nil){ return false, err }
if (localBlockchainNodeEnabled == true){
// We must have enabled the node after our earlier check
// Skip all remaining hosts.
return false, nil
}
if (len(addressesToUpdateList) == 0){
// No addresses to update remain.
return false, nil
}
//TODO: Add limit to size of below
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
if (err != nil){ return false, err }
if (hostProfileExists == false){
return false, nil
}
if (connectionEstablished == false){
return false, nil
}
defer peerClient.CloseConnection(connectionIdentifier)
successfulDownload, retrievedDepositObjectsList, addressesWithNoDepositsList, err := sendRequests.GetAddressDepositsFromHost(connectionIdentifier, hostIdentityHash, networkType, cryptocurrencyName, addressesToUpdateList)
if (err != nil) { return false, err }
if (successfulDownload == false){
err := logger.AddLogEntry("Network", "Failed to retrieve address deposits from host.")
if (err != nil) { return false, err }
// Skip to next host
return false, nil
}
err = trustedAddressDeposits.AddAddressDepositObjectsListToCache(hostIdentityHash, cryptocurrencyName, retrievedDepositObjectsList)
if (err != nil) { return false, err }
err = trustedAddressDeposits.AddAddressesWithNoDepositsToCache(hostIdentityHash, cryptocurrencyName, addressesWithNoDepositsList)
if (err != nil) { return false, err }
return true, nil
}
// We query each host for the identity deposits we want
// After each host, we get the new known/unknown identities list to make
// sure we do not query deposits which were unknown but have become known
hostsToQueryList, err := eligibleHosts.GetEligibleHostsList(networkType)
if (err != nil){ return err }
hostsQueried := 0
for _, hostIdentityHash := range hostsToQueryList{
downloadedAnyDeposits, err := downloadDepositsFromHost(hostIdentityHash)
if (err != nil) { return err }
if (downloadedAnyDeposits == true){
hostsQueried += 1
}
if (hostsQueried >= numberOfHostsToQuery){
break
}
}
return nil
}
Reference in a new issue View git blame Copy permalink