3372 lines
133 KiB
Go
3372 lines
133 KiB
Go
|
|
// queryHosts provides functions to download information from hosts
|
|
// These include parameters, profiles, messages, reports, reviews, identity deposits, and trusted moderation viewable statuses
|
|
// Each function will query a specified number of eligible hosts
|
|
|
|
package queryHosts
|
|
|
|
//TODO: If you download a user's identity, then the host is saying that the user's identity is funded. Add it to trustedFundedStatus
|
|
|
|
//TODO: Split requests up so that memory does not have to store all of the requested hashes
|
|
//TODO: Set a maximum limit, so that only a certain amount of data will be requested from each host
|
|
|
|
//TODO: Choose hosts more intelligently based on what ranges we are requesting
|
|
// Right now, we are sending requests to randomly chosen hosts until we reach the numberOfHostsToQuery
|
|
// This is a bad strategy, because, if we are requesting range 20-100, we might request from a host that is only hosting from range 10-25
|
|
// We should use numberOfHostsToQuery to choose a set of hosts that will host the entire range
|
|
// We must make sure that we sometimes request from hosts that only host a small number of total profiles.
|
|
|
|
// TODO: Add BroadcastToHosts, GetFundedStatuses, and more
|
|
|
|
//TODO: If only Host/Moderator mode is enabled, we can skip all of the downloaded identity/inbox fingerprinting prevention logic
|
|
// This logic is designed to prevent hosts from learning the user's moderator/host identity/behavior from their downloaded content
|
|
// For example, a user is moderating and hosting. requestors could determine a host's moderator identity by querying for
|
|
// profiles outside of their host range and seeing which mate profiles they have downloaded.
|
|
// Using this information, they could learn their moderator range and link it to their moderator identity/profile.
|
|
//
|
|
// We cannot disable fingerprinting if Mate mode is enabled, because a user's Mate criteria will change, so we cannot allow the host to know what their criteria was previously
|
|
// This is also true if a Host or Moderator's ranges change, but in those cases, the leak of privacy is fine
|
|
// This is because the information (previous host/moderator ranges) is not sensitive, and not a problem if it is leaked
|
|
|
|
import "seekia/internal/byteRange"
|
|
import "seekia/internal/badgerDatabase"
|
|
import "seekia/internal/contentMetadata"
|
|
import "seekia/internal/encoding"
|
|
import "seekia/internal/helpers"
|
|
import "seekia/internal/identity"
|
|
import "seekia/internal/logger"
|
|
import "seekia/internal/messaging/chatMessageStorage"
|
|
import "seekia/internal/moderation/moderatorScores"
|
|
import "seekia/internal/moderation/readReports"
|
|
import "seekia/internal/moderation/readReviews"
|
|
import "seekia/internal/moderation/reportStorage"
|
|
import "seekia/internal/moderation/reviewStorage"
|
|
import "seekia/internal/moderation/trustedAddressDeposits"
|
|
import "seekia/internal/moderation/trustedViewableStatus"
|
|
import "seekia/internal/network/eligibleHosts"
|
|
import "seekia/internal/network/hostRanges"
|
|
import "seekia/internal/network/maliciousHosts"
|
|
import "seekia/internal/network/mateCriteria"
|
|
import "seekia/internal/network/peerClient"
|
|
import "seekia/internal/network/sendRequests"
|
|
import "seekia/internal/network/serverResponse"
|
|
import "seekia/internal/parameters/parametersStorage"
|
|
import "seekia/internal/profiles/profileStorage"
|
|
import "seekia/internal/profiles/readProfiles"
|
|
|
|
import "bytes"
|
|
import "errors"
|
|
import "slices"
|
|
|
|
func DownloadParametersFromHosts(allowClearnet bool, networkType byte, numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadParametersFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
// Outputs:
|
|
// -bool: Successful download from host
|
|
// -error
|
|
downloadParametersFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
exists, hostIsHostingParameters, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "HostingParameters")
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
return false, errors.New("Database corrupt: Contains host Profile missing HostingParameters")
|
|
}
|
|
if (hostIsHostingParameters != "Yes"){
|
|
// This host is not hosting parameters, skip
|
|
return false, nil
|
|
}
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, err }
|
|
if (hostProfileExists == false){
|
|
return false, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, nil
|
|
}
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
downloadSuccessful, parametersInfoMap, err := sendRequests.GetParametersInfoFromHost(connectionIdentifier, hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
return false, nil
|
|
}
|
|
|
|
if (len(parametersInfoMap) == 0){
|
|
// The host has provided no parameters. We will not consider this a successful download
|
|
return false, nil
|
|
}
|
|
|
|
// We will check to see if our parameters broadcast time is older than the one the host is offering
|
|
// If it is, we will download it and continue
|
|
|
|
parametersTypesToRetrieveList := make([]string, 0)
|
|
|
|
for parametersType, parametersBroadcastTime := range parametersInfoMap{
|
|
|
|
storedParametersFound, _, _, storedParametersBroadcastTime, err := parametersStorage.GetAuthorizedParameters(parametersType, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (storedParametersFound == true){
|
|
if (storedParametersBroadcastTime < parametersBroadcastTime){
|
|
// The offered parameters are not newer than our existing parameters
|
|
// Skip downloading them.
|
|
continue
|
|
}
|
|
}
|
|
parametersTypesToRetrieveList = append(parametersTypesToRetrieveList, parametersType)
|
|
}
|
|
|
|
if (len(parametersTypesToRetrieveList) == 0){
|
|
// The host has no parameters that we need. Nothing to download.
|
|
return true, nil
|
|
}
|
|
|
|
// Now we download parameters
|
|
|
|
downloadSuccessful, receivedParametersList, err := sendRequests.GetParametersFromHost(connectionIdentifier, hostIdentityHash, networkType, parametersTypesToRetrieveList)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Unable to download parameters. Nothing left to do with this host.
|
|
return false, nil
|
|
}
|
|
|
|
if (len(receivedParametersList) == 0){
|
|
// Host is probably malicious
|
|
// Host could have received new admin permissions after we got parameters info
|
|
// Either way, we do not count this as a successful download
|
|
return false, nil
|
|
}
|
|
|
|
for _, parametersBytes := range receivedParametersList{
|
|
|
|
parametersAreValid, _, err := parametersStorage.AddParameters(parametersBytes)
|
|
if (err != nil) { return false, err }
|
|
if (parametersAreValid == false){
|
|
return false, errors.New("GetParametersFromHost not verifying parameters.")
|
|
}
|
|
}
|
|
|
|
// These parameters were downloaded, not necessarily authorized or added
|
|
|
|
numberOfDownloadedParametersString := helpers.ConvertIntToString(len(receivedParametersList))
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfDownloadedParametersString + " network parameters from host.")
|
|
if (err != nil) { return false, err }
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadParametersFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadProfilesFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
profileTypeToRetrieve string,
|
|
rangeToRetrieveStart [16]byte,
|
|
rangeToRetrieveEnd [16]byte,
|
|
identityHashesToRetrieveList [][16]byte,
|
|
criteria []byte,
|
|
getNewestProfilesOnly bool,
|
|
getViewableProfilesOnly bool,
|
|
downloadAllMode bool,
|
|
checkIfProfileShouldBeDownloaded func([28]byte, [16]byte, int64)(bool, error),
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadProfilesFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
// We cycle through each host
|
|
// We get the profiles info for profiles that are within our requested criteria/range/identitites list
|
|
// We see if any are knowingly outside of criteria, and download anyway if they are (to resist fingerprinting)
|
|
// We download the profiles.
|
|
// We don't download any profiles that we already have (if desiresPruningMode == false)
|
|
|
|
//TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable profiles
|
|
|
|
//Outputs:
|
|
// -bool: Successfully downloaded from host (false if host does not have any profiles we need)
|
|
// -error
|
|
downloadProfilesFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
hostIsHostingProfileType, theirHostRangeStart, theirHostRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, profileTypeToRetrieve)
|
|
if (err != nil) { return false, err }
|
|
if (hostIsHostingProfileType == false){
|
|
// Host is not hosting any profiles of this profileType. Skip to next host
|
|
return false, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -[21]byte: Connection identifier
|
|
// -[]serverResponse.ProfileInfoStruct: Retrieved profiles info objects list
|
|
// -error
|
|
getRetrievedProfileInfoObjectsList := func()(bool, [21]byte, []serverResponse.ProfileInfoStruct, error){
|
|
|
|
// Our request will either be based on identity hash range or identity hash list
|
|
// We will split up the request into either ranges or identity hashes
|
|
|
|
maximumProfilesInGetProfilesInfoRequest := serverResponse.MaximumProfilesInResponse_GetProfilesInfo - 100
|
|
|
|
if (len(identityHashesToRetrieveList) != 0){
|
|
|
|
// We will retrieve identities lists, not identity ranges
|
|
|
|
anyValuesExist, identitiesInMyRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identityHashesToRetrieveList)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
if (anyValuesExist == false){
|
|
// This should not happen
|
|
return false, [21]byte{}, nil, errors.New("Identity hashes list to retrieve contains no identities within request range.")
|
|
}
|
|
|
|
anyValuesExist, identitiesInTheirRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostRangeStart, theirHostRangeEnd, identitiesInMyRangeList)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
if (anyValuesExist == false){
|
|
// This host is not hosting any identities we desire. We will skip them.
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
// Now we split request into sublists if needed
|
|
|
|
getMaximumIdentitiesPerRequest := func()int{
|
|
|
|
if (getNewestProfilesOnly == true){
|
|
return maximumProfilesInGetProfilesInfoRequest
|
|
}
|
|
|
|
// We assume each identity has approximately 3 profiles
|
|
|
|
maximumIdentitiesPerRequest := maximumProfilesInGetProfilesInfoRequest / 3
|
|
|
|
return maximumIdentitiesPerRequest
|
|
}
|
|
|
|
maximumIdentitiesPerRequest := getMaximumIdentitiesPerRequest()
|
|
|
|
sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInTheirRangeList, maximumIdentitiesPerRequest)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
downloadProfileInfoObjectsList := func()(bool, []serverResponse.ProfileInfoStruct, error){
|
|
|
|
retrievedProfileInfoObjectsList := make([]serverResponse.ProfileInfoStruct, 0)
|
|
|
|
minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds()
|
|
|
|
for _, identityHashesSublist := range sublistsToQueryList{
|
|
|
|
downloadSuccessful, profilesInfoObjectsList, err := sendRequests.GetProfilesInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist, criteria, getNewestProfilesOnly, getViewableProfilesOnly)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this subrange.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, cease connection to host
|
|
err := logger.AddLogEntry("Network", "Failed to download profiles info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
err = peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
retrievedProfileInfoObjectsList = append(retrievedProfileInfoObjectsList, profilesInfoObjectsList...)
|
|
}
|
|
|
|
return true, retrievedProfileInfoObjectsList, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedProfileInfoObjectsList, err := downloadProfileInfoObjectsList()
|
|
if (err != nil){
|
|
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err = peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedProfileInfoObjectsList, nil
|
|
}
|
|
|
|
// We will retrieve identity ranges, not identity lists
|
|
|
|
//Outputs:
|
|
// -Any intersection found
|
|
// -[16]byte: Intersection range start
|
|
// -[16]byte: Intersection range end
|
|
// -error
|
|
getIntersectionRange := func()(bool, [16]byte, [16]byte, error){
|
|
|
|
if (profileTypeToRetrieve != "Mate"){
|
|
// Hosts must host all or none of host/moderator identities, so intersection is always all our requested identities
|
|
|
|
return true, rangeToRetrieveStart, rangeToRetrieveEnd, nil
|
|
}
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostRangeStart, theirHostRangeEnd)
|
|
if (err != nil) { return false, [16]byte{}, [16]byte{}, err }
|
|
|
|
if (anyIntersectionFound == false){
|
|
|
|
return false, [16]byte{}, [16]byte{}, nil
|
|
}
|
|
|
|
return true, intersectionRangeStart, intersectionRangeEnd, nil
|
|
}
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := getIntersectionRange()
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
if (anyIntersectionFound == false){
|
|
// Host is not hosting any profiles within the range that we are requesting
|
|
// Skip this host
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
exists, hostedProfilesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, profileTypeToRetrieve + "ProfilesQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, errors.New("Database corrupt: Contains host profile missing profilesQuantity")
|
|
}
|
|
|
|
hostedProfilesQuantityInt64, err := helpers.ConvertStringToInt64(hostedProfilesQuantity)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
|
|
if (hostedProfilesQuantityInt64 == 0){
|
|
// Host must have just started their node. Skip them.
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
estimatedNumItemsInSubrange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostRangeStart, theirHostRangeEnd, hostedProfilesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
|
|
subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInSubrange, int64(maximumProfilesInGetProfilesInfoRequest))
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -[]serverResponse.ProfileInfoStruct: Retrieved profiles info map list
|
|
// -error
|
|
downloadProfileInfoObjectsList := func()(bool, []serverResponse.ProfileInfoStruct, error){
|
|
|
|
retrievedProfileInfoObjectsList := make([]serverResponse.ProfileInfoStruct, 0)
|
|
|
|
for _, subrangeObject := range subrangesToQueryList{
|
|
|
|
subrangeStart := subrangeObject.SubrangeStart
|
|
subrangeEnd := subrangeObject.SubrangeEnd
|
|
|
|
downloadSuccessful, profileInfoObjectsList, err := sendRequests.GetProfilesInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, subrangeStart, subrangeEnd, identityHashesToRetrieveList, criteria, getNewestProfilesOnly, getViewableProfilesOnly)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this subrange.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, close connection
|
|
err := logger.AddLogEntry("Network", "Failed to download profiles info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
retrievedProfileInfoObjectsList = append(retrievedProfileInfoObjectsList, profileInfoObjectsList...)
|
|
}
|
|
|
|
return true, retrievedProfileInfoObjectsList, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedProfileInfoObjectsList, err := downloadProfileInfoObjectsList()
|
|
if (err != nil){
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
err = peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedProfileInfoObjectsList, nil
|
|
}
|
|
|
|
downloadSuccessful, connectionIdentifier, retrievedProfilesInfoObjectsList, err := getRetrievedProfileInfoObjectsList()
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
if (len(retrievedProfilesInfoObjectsList) == 0){
|
|
// Host has no profiles that we need. We are done with this host.
|
|
|
|
return true, nil
|
|
}
|
|
|
|
profileHashesToDownloadList := make([][28]byte, 0)
|
|
|
|
// Structure: Profile Hash -> Expected Broadcast Time
|
|
profileBroadcastTimesMap := make(map[[28]byte]int64)
|
|
|
|
// Structure: Profile Hash -> Identity Hash
|
|
profileIdentityHashesMap := make(map[[28]byte][16]byte)
|
|
|
|
hostIsMalicious := false
|
|
|
|
for _, profileInfoObject := range retrievedProfilesInfoObjectsList{
|
|
|
|
receivedProfileHash := profileInfoObject.ProfileHash
|
|
|
|
receivedProfileAuthor := profileInfoObject.ProfileAuthor
|
|
|
|
receivedProfileBroadcastTime := profileInfoObject.ProfileBroadcastTime
|
|
|
|
// Now we check to see if we already have the profile
|
|
// We check to see if it is known to be outside of requested range/identities list/criteria (host is malicious)
|
|
// If it is, we still request it in our download, and then set host as malicious afterwards
|
|
// Otherwise, host would know that we have this profile stored
|
|
//TODO: Disable this step if only host/moderator mode is enabled
|
|
|
|
storedProfileExists, storedProfileBytes, err := badgerDatabase.GetUserProfile(profileTypeToRetrieve, receivedProfileHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
checkIfHostIsOfferingInvalidProfile := func()(bool, error){
|
|
|
|
if (storedProfileExists == false){
|
|
return false, nil
|
|
}
|
|
|
|
ableToRead, storedProfileHash, profileVersion, profileNetworkType, storedProfileAuthor, storedProfileBroadcastTime, _, storedRawProfileMap, err := readProfiles.ReadProfileAndHash(false, storedProfileBytes)
|
|
if (err != nil) { return false, err }
|
|
if (ableToRead == false){
|
|
return false, errors.New("Database corrupt: Contains invalid profile.")
|
|
}
|
|
if (storedProfileHash != receivedProfileHash){
|
|
return false, errors.New("Database corrupt: Contains profile with different hash than entry key.")
|
|
}
|
|
if (profileNetworkType != networkType){
|
|
return true, nil
|
|
}
|
|
if (storedProfileAuthor != receivedProfileAuthor){
|
|
return true, nil
|
|
}
|
|
if (storedProfileBroadcastTime != receivedProfileBroadcastTime){
|
|
return true, nil
|
|
}
|
|
|
|
if (criteria != nil){
|
|
criteriaIsValid, fulfillsCriteria, err := mateCriteria.CheckIfMateProfileFulfillsCriteria(false, profileVersion, storedRawProfileMap, criteria)
|
|
if (err != nil) { return false, err }
|
|
if (criteriaIsValid == false){
|
|
return false, errors.New("DownloadProfilesFromHosts called with invalid criteria.")
|
|
}
|
|
if (fulfillsCriteria == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
hostIsOfferingInvalidProfile, err := checkIfHostIsOfferingInvalidProfile()
|
|
if (err != nil) { return false, err }
|
|
|
|
if (hostIsOfferingInvalidProfile == true){
|
|
// Host is malicious, we must still download the profile
|
|
hostIsMalicious = true
|
|
|
|
profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash)
|
|
profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor
|
|
profileBroadcastTimesMap[receivedProfileHash] = receivedProfileBroadcastTime
|
|
|
|
continue
|
|
}
|
|
|
|
if (downloadAllMode == true){
|
|
// downloadAllMode is enabled, we must download all profiles regardless of if we already have it
|
|
// This is done to prevent hosts learning our private desires
|
|
profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash)
|
|
profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor
|
|
profileBroadcastTimesMap[receivedProfileHash] = receivedProfileBroadcastTime
|
|
continue
|
|
}
|
|
|
|
if (storedProfileExists == true){
|
|
|
|
// We already have profile, skip downloading it
|
|
|
|
if (getViewableProfilesOnly == true){
|
|
|
|
err = trustedViewableStatus.AddTrustedProfileIsViewableStatus(receivedProfileHash, hostIdentityHash, true)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
// Profile does not exist, we must check if we want to download it
|
|
// This works differently for Hosting, Moderation, and Browsing
|
|
|
|
shouldDownload, err := checkIfProfileShouldBeDownloaded(receivedProfileHash, receivedProfileAuthor, receivedProfileBroadcastTime)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (shouldDownload == true){
|
|
profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash)
|
|
profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor
|
|
profileBroadcastTimesMap[receivedProfileHash] = receivedProfileBroadcastTime
|
|
continue
|
|
}
|
|
}
|
|
|
|
if (len(profileHashesToDownloadList) == 0){
|
|
// Nothing to download, done with this host.
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// Now we retrieve profiles from host
|
|
|
|
maximumProfileHashesInResponse, err := serverResponse.GetMaximumProfilesInResponse_GetProfiles(profileTypeToRetrieve)
|
|
if (err != nil) { return false, err }
|
|
|
|
profileHashesSublistsList, err := helpers.SplitListIntoSublists(profileHashesToDownloadList, maximumProfileHashesInResponse)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfDownloadedProfiles := 0
|
|
|
|
for _, profileHashesSublist := range profileHashesSublistsList{
|
|
|
|
downloadSuccessful, profilesList, err := sendRequests.GetProfilesFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, profileHashesSublist, profileIdentityHashesMap, profileBroadcastTimesMap, nil)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Request has failed
|
|
// We will stop downloading from this host
|
|
break
|
|
}
|
|
|
|
for _, profileBytes := range profilesList{
|
|
|
|
//TODO: If downloadAll mode is enabled, we need to check to see if profile should be imported or not (desires pruning mode)
|
|
|
|
profileIsWellFormed, addedProfileHash, err := profileStorage.AddUserProfile(profileBytes)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (profileIsWellFormed == false){
|
|
return false, errors.New("GetProfilesFromHost not verifying profiles are well formed.")
|
|
}
|
|
|
|
if (getViewableProfilesOnly == true){
|
|
|
|
err = trustedViewableStatus.AddTrustedProfileIsViewableStatus(addedProfileHash, hostIdentityHash, true)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
}
|
|
|
|
numberOfDownloadedProfiles += len(profilesList)
|
|
}
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
if (numberOfDownloadedProfiles != 0){
|
|
|
|
numberOfProfilesDownloadedString := helpers.ConvertIntToString(numberOfDownloadedProfiles)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfProfilesDownloadedString + " " + profileTypeToRetrieve + " profiles from host.")
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadProfilesFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadMessagesFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
rangeToRetrieveStart [10]byte,
|
|
rangeToRetrieveEnd [10]byte,
|
|
inboxesToRetrieveList [][10]byte,
|
|
getViewableMessagesOnly bool,
|
|
getDecryptableMessagesOnly bool,
|
|
checkIfMessageShouldBeDownloaded func([26]byte)(bool, error),
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadMessagesFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
// We cycle through each host
|
|
// We get all message hashes that they host that are within our requested range/inboxes list
|
|
// We see if any are knowingly outside of inboxes list, and download anyway if they are (to resist fingerprinting)
|
|
// We download the messages that we don't already have
|
|
|
|
//TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable messages
|
|
|
|
//Outputs:
|
|
// -bool: Successfully downloaded from host (false if host does not have any messages we need)
|
|
// -error
|
|
downloadMessagesFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
hostIsHostingMessages, theirInboxRangeStart, theirInboxRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap)
|
|
if (hostIsHostingMessages == false){
|
|
// Host is not hosting any messages. Skip to next host
|
|
return false, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -[21]byte: Connection identifier
|
|
// -[][26]byte: Retrieved message hashes list
|
|
// -[10]byte: Inbox Range start of requested messages
|
|
// -[10]byte: Inbox range end of requested messages
|
|
// -[][10]byte: Requested inboxes list (nil if none were requested)
|
|
// -error
|
|
getReceivedMessageHashesList := func()(bool, [21]byte, [][26]byte, [10]byte, [10]byte, [][10]byte, error){
|
|
|
|
// Our request will either be based on inbox range or inbox list
|
|
// We will split up the request into either ranges or inboxes
|
|
|
|
exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile missing MessagesQuantity")
|
|
}
|
|
|
|
hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
maximumMessageHashesInResponse := serverResponse.MaximumMessageHashesInResponse_GetMessageHashesList - 100
|
|
|
|
if (len(inboxesToRetrieveList) != 0){
|
|
|
|
// We will split request into inbox lists, not inbox ranges
|
|
|
|
anyValuesExist, inboxesInMyRangeList, err := byteRange.GetAllInboxesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, inboxesToRetrieveList)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
if (anyValuesExist == false){
|
|
// This should not happen
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Inboxes list to retrieve contains no inboxes within request range.")
|
|
}
|
|
|
|
anyValuesExist, inboxesInTheirRangeList, err := byteRange.GetAllInboxesInListWithinRange(theirInboxRangeStart, theirInboxRangeEnd, inboxesInMyRangeList)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
if (anyValuesExist == false){
|
|
// This host is not hosting any inboxes we desire. We will skip them.
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
// Now we split request into sublists if needed
|
|
|
|
exists, hostedInboxesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "InboxesQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile missing InboxesQuantity")
|
|
}
|
|
|
|
hostedInboxesQuantityInt64, err := helpers.ConvertStringToInt64(hostedInboxesQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid InboxesQuantity: " + hostedInboxesQuantity)
|
|
}
|
|
|
|
if (hostedInboxesQuantityInt64 == 0){
|
|
// Host has no inboxes
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
estimatedNumMessagesPerInbox := hostedMessagesQuantityInt64/hostedInboxesQuantityInt64
|
|
|
|
maximumInboxesPerRequest := int64(maximumMessageHashesInResponse) / estimatedNumMessagesPerInbox
|
|
|
|
maximumInboxesPerRequestInt, err := helpers.ConvertInt64ToInt(maximumInboxesPerRequest)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
sublistsToQueryList, err := helpers.SplitListIntoSublists(inboxesInTheirRangeList, maximumInboxesPerRequestInt)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds()
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -[][26]byte: Message hashes list
|
|
// -error
|
|
getMessageHashesListFromHost := func()(bool, [][26]byte, error){
|
|
|
|
retrievedMessageHashesList := make([][26]byte, 0)
|
|
|
|
for _, inboxesSublist := range sublistsToQueryList{
|
|
|
|
downloadSuccessful, messageHashesList, err := sendRequests.GetMessageHashesListFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, inboxesSublist, getViewableMessagesOnly, getDecryptableMessagesOnly)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// We are done with this host
|
|
err := logger.AddLogEntry("Network", "Failed to download message hashes from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
retrievedMessageHashesList = append(retrievedMessageHashesList, messageHashesList...)
|
|
}
|
|
|
|
return true, retrievedMessageHashesList, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedMessageHashesList, err := getMessageHashesListFromHost()
|
|
if (err != nil){
|
|
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedMessageHashesList, minimumRange, maximumRange, inboxesInTheirRangeList, nil
|
|
}
|
|
|
|
// We will retrieve using inbox ranges, not inbox lists
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxRangeStart, theirInboxRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
if (anyIntersectionFound == false){
|
|
// Host is not hosting any messages within the inboxes range that we are requesting
|
|
// Skip this host
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxRangeStart, theirInboxRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, int64(maximumMessageHashesInResponse))
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
getMessageHashesListFromHost := func()(bool, [][26]byte, error){
|
|
|
|
retrievedMessageHashesList := make([][26]byte, 0)
|
|
|
|
for _, subrangeObject := range subrangesToQueryList{
|
|
|
|
subrangeStart := subrangeObject.SubrangeStart
|
|
subrangeEnd := subrangeObject.SubrangeEnd
|
|
|
|
downloadSuccessful, messageHashesList, err := sendRequests.GetMessageHashesListFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, inboxesToRetrieveList, getViewableMessagesOnly, getDecryptableMessagesOnly)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this subrange.
|
|
//Could be that host went offline permanently or just this connection failed.
|
|
// Either way, end connection to host
|
|
err := logger.AddLogEntry("Network", "Failed to download message hashes from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
retrievedMessageHashesList = append(retrievedMessageHashesList, messageHashesList...)
|
|
}
|
|
|
|
return true, retrievedMessageHashesList, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedMessageHashesList, err := getMessageHashesListFromHost()
|
|
if (err != nil){
|
|
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil
|
|
}
|
|
|
|
emptyList := make([][10]byte, 0)
|
|
|
|
return true, connectionIdentifier, retrievedMessageHashesList, intersectionRangeStart, intersectionRangeEnd, emptyList, nil
|
|
}
|
|
|
|
downloadSuccessful, connectionIdentifier, retrievedMessageHashesList, requestedInboxesRangeStart, requestedInboxesRangeEnd, requestedInboxesList, err := getReceivedMessageHashesList()
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
|
|
return false, nil
|
|
}
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
// Now we see if we have any of these messages, and if they do not fulfill the request range/inboxes list
|
|
// If they do not fulfill the request, we know host is malicious
|
|
// We still download the messages to prevent host from knowing that we have those messages downloaded
|
|
// We then mark the host as malicious
|
|
//TODO: Immediately stop connection to host if only Host/Moderator mode is enabled, because fingerprinting is not a risk.
|
|
|
|
messageHashesToDownloadList := make([][26]byte, 0)
|
|
|
|
hostIsMalicious := false
|
|
|
|
for _, receivedMessageHash := range retrievedMessageHashesList{
|
|
|
|
checkIfHostIsOfferingInvalidMessage := func()(bool, error){
|
|
|
|
messageMetadataExists, _, messageNetworkType, _, messageInbox, _, err := contentMetadata.GetMessageMetadata(receivedMessageHash)
|
|
if (err != nil) { return false, err }
|
|
if (messageMetadataExists == false){
|
|
return false, nil
|
|
}
|
|
if (messageNetworkType != networkType){
|
|
return true, nil
|
|
}
|
|
|
|
if (len(requestedInboxesList) != 0){
|
|
isInInboxesList := slices.Contains(requestedInboxesList, messageInbox)
|
|
if (isInInboxesList == false){
|
|
return true, nil
|
|
}
|
|
} else {
|
|
|
|
inboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(requestedInboxesRangeStart, requestedInboxesRangeEnd, messageInbox)
|
|
if (err != nil) { return false, err }
|
|
if (inboxIsWithinRange == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
hostIsOfferingInvalidMessage, err := checkIfHostIsOfferingInvalidMessage()
|
|
if (err != nil) { return false, err }
|
|
|
|
if (hostIsOfferingInvalidMessage == true){
|
|
// Host is malicious, we must still download the message so they do not learn that we have this message downloaded (or did at one point)
|
|
hostIsMalicious = true
|
|
|
|
messageHashesToDownloadList = append(messageHashesToDownloadList, receivedMessageHash)
|
|
continue
|
|
}
|
|
|
|
storedMessageExists, _, err := badgerDatabase.GetChatMessage(receivedMessageHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (storedMessageExists == true){
|
|
|
|
// We already have the message, skip downloading it
|
|
continue
|
|
}
|
|
|
|
// Message does not exist, we must check if we want to download it
|
|
// This works differently for Hosting, Moderation, and Viewing
|
|
// For instance, if the user is downloading messages for viewing, they will not redownload profiles that they have already imported/deleted
|
|
|
|
shouldDownload, err := checkIfMessageShouldBeDownloaded(receivedMessageHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (shouldDownload == true){
|
|
messageHashesToDownloadList = append(messageHashesToDownloadList, receivedMessageHash)
|
|
continue
|
|
}
|
|
}
|
|
|
|
if (len(messageHashesToDownloadList) == 0){
|
|
// Nothing to download, done with this host.
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Now we retrieve messages from host
|
|
|
|
maximumMessagesInResponse := serverResponse.MaximumMessagesInResponse_GetMessages
|
|
|
|
messageHashesSublistsList, err := helpers.SplitListIntoSublists(messageHashesToDownloadList, maximumMessagesInResponse)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfDownloadedMessages := 0
|
|
|
|
for _, messageHashesSublist := range messageHashesSublistsList{
|
|
|
|
downloadSuccessful, messagesList, err := sendRequests.GetMessagesFromHost(connectionIdentifier, hostIdentityHash, networkType, messageHashesSublist, requestedInboxesRangeStart, requestedInboxesRangeEnd, requestedInboxesList)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Request has failed
|
|
// We will stop downloading from this host
|
|
break
|
|
}
|
|
|
|
for _, messageBytes := range messagesList{
|
|
|
|
messageIsWellFormed, err := chatMessageStorage.AddMessage(messageBytes)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (messageIsWellFormed == false){
|
|
return false, errors.New("GetMessagesFromHost not verifying messages are well formed.")
|
|
}
|
|
}
|
|
|
|
numberOfDownloadedMessages += len(messagesList)
|
|
}
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
if (numberOfDownloadedMessages != 0){
|
|
numberOfMessagesDownloadedString := helpers.ConvertIntToString(numberOfDownloadedMessages)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfMessagesDownloadedString + " messages from host.")
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadMessagesFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadIdentityReviewsFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
identityTypeToRetrieve string,
|
|
rangeToRetrieveStart [16]byte,
|
|
rangeToRetrieveEnd [16]byte,
|
|
reviewedIdentityHashesToRetrieveList [][16]byte,
|
|
reviewersToRetrieveList [][16]byte,
|
|
checkIfReviewShouldBeDownloaded func([29]byte, []byte)(bool, error),
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadIdentityReviewsFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
//TODO: Verify inputs
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -error
|
|
downloadReviewsFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
// Outputs:
|
|
// -bool: Successfully downloaded from host
|
|
// -[21]byte: Connection identifier
|
|
// -map[[29]byte][]byte: Map of review info: ReviewHash -> ReviewedHash
|
|
// -[16]byte: Expected Identity Start Range of retrieved reviews
|
|
// -[16]byte: Expected Identity End range of retrieved reviews
|
|
// -[][16]byte: Expected retrieved identity hashes list
|
|
// -error
|
|
getRetrievedReviewsInfoMap := func()(bool, [21]byte, map[[29]byte][]byte, [16]byte, [16]byte, [][16]byte, error){
|
|
|
|
hostIsHostingIdentityType, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, identityTypeToRetrieve)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostIsHostingIdentityType == false){
|
|
// Host is not hosting any profiles/reviews of this identityType. Skip to next host
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
exists, identityTypeIdentitiesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, identityTypeToRetrieve + "IdentitiesQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "IdentitiesQuantity")
|
|
}
|
|
|
|
identityTypeIdentitiesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeIdentitiesQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity)
|
|
}
|
|
|
|
getMaximumIdentitiesPerRequest := func()(int64, error){
|
|
|
|
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetIdentityReviewsInfo
|
|
|
|
if (len(reviewersToRetrieveList) != 0){
|
|
|
|
// We assume each reviewer has reviewed each identity and all it's profiles
|
|
|
|
profilesQuantityAttributeName := identityTypeToRetrieve + "ProfilesQuantity"
|
|
|
|
exists, identityTypeProfilesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, profilesQuantityAttributeName)
|
|
if (err != nil) { return 0, err }
|
|
if (exists == false){
|
|
return 0, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ProfilesQuantity")
|
|
}
|
|
|
|
identityTypeProfilesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeProfilesQuantity)
|
|
if (err != nil) { return 0, err }
|
|
|
|
estimatedNumberOfProfilesPerIdentity := identityTypeProfilesQuantityInt64 / identityTypeIdentitiesQuantityInt64
|
|
|
|
numberOfReviewersToRetrieve := int64(len(reviewersToRetrieveList))
|
|
|
|
estimatedReviewsPerIdentity := numberOfReviewersToRetrieve * (estimatedNumberOfProfilesPerIdentity + 1)
|
|
|
|
maximumIdentitiesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerIdentity
|
|
|
|
return maximumIdentitiesPerRequest, nil
|
|
}
|
|
|
|
reviewsQuantityAttributeName := identityTypeToRetrieve + "ReviewsQuantity"
|
|
|
|
exists, identityTypeReviewsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, reviewsQuantityAttributeName)
|
|
if (err != nil) { return 0, err }
|
|
if (exists == false){
|
|
return 0, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ReviewsQuantity")
|
|
}
|
|
|
|
identityTypeReviewsQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeReviewsQuantity)
|
|
if (err != nil) { return 0, err }
|
|
|
|
estimatedReviewsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReviewsQuantityInt64
|
|
|
|
maximumIdentitiesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerIdentity
|
|
|
|
return maximumIdentitiesPerRequest, nil
|
|
}
|
|
|
|
maximumIdentitiesPerRequest, err := getMaximumIdentitiesPerRequest()
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
// Our request will either be based on identity hash range or identity hashes list
|
|
// We will split up the request into either ranges or identity hashes
|
|
|
|
if (len(reviewedIdentityHashesToRetrieveList) != 0){
|
|
|
|
// We will split the request into identity hash lists
|
|
|
|
hostingAny, identitiesInHostRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, reviewedIdentityHashesToRetrieveList)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostingAny == false){
|
|
// Host is not hosting any of our desired identities. Skip host.
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
// Below step is just being overly cautious.
|
|
// The GetIdentityReviewsfromHosts function should request either a list of identities or a range. Not both.
|
|
hostingAny, identitiesInRequestRange, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identitiesInHostRangeList)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostingAny == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Requested review identity hashes list outside of request range")
|
|
}
|
|
|
|
sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInRequestRange, int(maximumIdentitiesPerRequest))
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds()
|
|
|
|
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][]byte, error){
|
|
|
|
retrievedReviewsInfoMap := make(map[[29]byte][]byte)
|
|
|
|
for _, identityHashesSublist := range sublistsToQueryList{
|
|
|
|
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetIdentityReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist, reviewersToRetrieveList)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, stop download from host.
|
|
err := logger.AddLogEntry("Network", "Failed to download identity reviews info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
|
|
retrievedReviewsInfoMap[reviewHash] = reviewedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReviewsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReviewsInfoMap, minimumRange, maximumRange, identitiesInRequestRange, nil
|
|
}
|
|
|
|
// We will retrieve based on identity range, not identities list
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (anyIntersectionFound == false){
|
|
// Host is not hosting any reviews within the identites range that we are requesting
|
|
// Skip this host
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
estimatedNumIdentitiesInIntersectionRange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, identityTypeIdentitiesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumIdentitiesInIntersectionRange, maximumIdentitiesPerRequest)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][]byte, error){
|
|
|
|
retrievedReviewsInfoMap := make(map[[29]byte][]byte)
|
|
|
|
for _, subrangeObject := range subrangesToQueryList{
|
|
|
|
subrangeStart := subrangeObject.SubrangeStart
|
|
subrangeEnd := subrangeObject.SubrangeEnd
|
|
|
|
emptyList := make([][16]byte, 0)
|
|
|
|
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetIdentityReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, subrangeStart, subrangeEnd, emptyList, reviewersToRetrieveList)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, stop download.
|
|
err := logger.AddLogEntry("Network", "Failed to download identity reviews info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
|
|
retrievedReviewsInfoMap[reviewHash] = reviewedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReviewsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReviewsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil, nil
|
|
}
|
|
|
|
hostDownloadSuccessful, connectionIdentifier, retrievedReviewsInfoMap, expectedRangeStart, expectedRangeEnd, expectedReviewedIdentityHashesList, err := getRetrievedReviewsInfoMap()
|
|
if (err != nil) { return false, err }
|
|
if (hostDownloadSuccessful == false){
|
|
// We did not download any reviews from this host. Skip them.
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
// Now we see if we have any of these reviews, and if they do not fulfill the requests parameters
|
|
// If they do not fulfill the request, we know host is malicious
|
|
// This requires checking against content stored in our database
|
|
// A host could use this to learn information about what content we are storing
|
|
|
|
// We still download the reviews to prevent host from knowing that we have the reviewed content downloaded
|
|
// Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response
|
|
// Only after the full request is finished do we mark the host as malicious
|
|
|
|
// TODO: We shouldn't have to wait until the end of the request if we are only in Host/Moderator mode
|
|
// This is because we don't need to worry about leaking what content we have stored
|
|
|
|
reviewHashesToDownloadList := make([][29]byte, 0)
|
|
|
|
hostIsMalicious := false
|
|
|
|
for receivedReviewHash, receivedReviewedHash := range retrievedReviewsInfoMap{
|
|
|
|
storedReviewExists, storedReviewBytes, err := badgerDatabase.GetReview(receivedReviewHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
checkIfHostIsOfferingInvalidReview := func()(bool, error){
|
|
|
|
if (storedReviewExists == true){
|
|
|
|
ableToRead, storedReviewHash, _, reviewNetworkType, storedReviewReviewerIdentity, _, storedReviewType, storedReviewReviewedHash, _, _, err := readReviews.ReadReviewAndHash(false, storedReviewBytes)
|
|
if (err != nil) { return false, err }
|
|
if (ableToRead == false){
|
|
return false, errors.New("Database corrupt: Contains invalid review.")
|
|
}
|
|
if (storedReviewHash != receivedReviewHash){
|
|
return false, errors.New("Database corrupt: Contains review with invalid entry key hash")
|
|
}
|
|
if (reviewNetworkType != networkType){
|
|
return true, nil
|
|
}
|
|
|
|
if (len(reviewersToRetrieveList) != 0){
|
|
|
|
isInList := slices.Contains(reviewersToRetrieveList, storedReviewReviewerIdentity)
|
|
if (isInList == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
areEqual := bytes.Equal(storedReviewReviewedHash, receivedReviewedHash)
|
|
if (areEqual == false){
|
|
return true, nil
|
|
}
|
|
|
|
if (storedReviewType != "Identity" && storedReviewType != "Profile" && storedReviewType != "Attribute"){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
// Now we check to see if we are aware of the reviewed content's metadata, and if it matches our request parameters
|
|
|
|
receivedReviewedType, err := helpers.GetReviewedTypeFromReviewedHash(receivedReviewedHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (receivedReviewedType == "Profile"){
|
|
|
|
// We must verify that the profile's identity hash is expected
|
|
// We don't have to do this for identities, because we already checked that in the GetIdentityReviewsInfo function
|
|
|
|
if (len(receivedReviewedHash) != 28){
|
|
receivedReviewedHashHex := encoding.EncodeBytesToHexString(receivedReviewedHash)
|
|
return false, errors.New("GetReviewedTypeFromReviewedHash returning Profile for different length reviewedHash: " + receivedReviewedHashHex)
|
|
}
|
|
|
|
receivedReviewedProfileHash := [28]byte(receivedReviewedHash)
|
|
|
|
metadataIsKnown, _, profileNetworkType, reviewedProfileAuthor, _, profileIsDisabled, _, _, err := contentMetadata.GetProfileMetadata(receivedReviewedProfileHash)
|
|
if (err != nil) { return false, err }
|
|
if (metadataIsKnown == false){
|
|
return false, nil
|
|
}
|
|
if (profileIsDisabled == true){
|
|
// Disabled profiles cannot be reviewed. Host must be malicious.
|
|
return true, nil
|
|
}
|
|
if (profileNetworkType != networkType){
|
|
// Review is reviewing profile from different networkType
|
|
// Host should have checked this already. Host must be malicious.
|
|
return true, nil
|
|
}
|
|
|
|
if (expectedReviewedIdentityHashesList != nil){
|
|
|
|
identityIsInList := slices.Contains(expectedReviewedIdentityHashesList, reviewedProfileAuthor)
|
|
if (identityIsInList == false){
|
|
return true, nil
|
|
}
|
|
} else {
|
|
|
|
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, reviewedProfileAuthor)
|
|
if (err != nil) { return false, err }
|
|
if (identityIsWithinRange == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
}
|
|
if (receivedReviewedType == "Attribute"){
|
|
|
|
if (len(receivedReviewedHash) != 27){
|
|
receivedReviewedHashHex := encoding.EncodeBytesToHexString(receivedReviewedHash)
|
|
return false, errors.New("GetReviewedTypeFromReviewedHash returning Attribute for different length reviewedHash: " + receivedReviewedHashHex)
|
|
}
|
|
|
|
receivedReviewedAttributeHash := [27]byte(receivedReviewedHash)
|
|
|
|
metadataExists, _, attributeAuthorIdentityHash, attributeNetworkType, _, err := profileStorage.GetProfileAttributeMetadata(receivedReviewedAttributeHash)
|
|
if (err != nil){ return false, err }
|
|
if (metadataExists == true){
|
|
|
|
if (expectedReviewedIdentityHashesList != nil){
|
|
|
|
identityIsInList := slices.Contains(expectedReviewedIdentityHashesList, attributeAuthorIdentityHash)
|
|
if (identityIsInList == false){
|
|
return true, nil
|
|
}
|
|
} else {
|
|
|
|
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, attributeAuthorIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
if (identityIsWithinRange == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
if (attributeNetworkType != networkType){
|
|
// Review is reviewing attribute from different networkType
|
|
// Host should have checked this already. Host must be malicious.
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
// Review is not invalid, as far as we know
|
|
|
|
return false, nil
|
|
}
|
|
|
|
hostIsOfferingInvalidReview, err := checkIfHostIsOfferingInvalidReview()
|
|
if (err != nil) { return false, err }
|
|
if (hostIsOfferingInvalidReview == true){
|
|
// Host is malicious, we must still download the review
|
|
hostIsMalicious = true
|
|
|
|
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
|
|
continue
|
|
}
|
|
|
|
if (storedReviewExists == true){
|
|
|
|
// We already have the review, skip downloading it
|
|
continue
|
|
}
|
|
|
|
// Review does not exist, we must check if we want to download it
|
|
|
|
shouldDownload, err := checkIfReviewShouldBeDownloaded(receivedReviewHash, receivedReviewedHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (shouldDownload == true){
|
|
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
|
|
continue
|
|
}
|
|
}
|
|
|
|
if (len(reviewHashesToDownloadList) == 0){
|
|
// Nothing to download, we are done with this host.
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Now we download reviews
|
|
|
|
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetReviews
|
|
|
|
reviewHashesSublistsList, err := helpers.SplitListIntoSublists(reviewHashesToDownloadList, maximumReviewsInResponse)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfDownloadedReviews := 0
|
|
|
|
for _, reviewHashesSublist := range reviewHashesSublistsList{
|
|
|
|
downloadSuccessful, reviewsList, err := sendRequests.GetReviewsFromHost(connectionIdentifier, hostIdentityHash, networkType, reviewHashesSublist, retrievedReviewsInfoMap, reviewersToRetrieveList)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Request has failed
|
|
// We will stop downloading from this host
|
|
break
|
|
}
|
|
if (hostIsMalicious == true){
|
|
// We don't add reviews from malicious hosts
|
|
continue
|
|
}
|
|
|
|
for _, reviewBytes := range reviewsList{
|
|
|
|
reviewIsWellFormed, err := reviewStorage.AddReview(reviewBytes)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (reviewIsWellFormed == false){
|
|
return false, errors.New("GetReviewsFromHost not verifying reviews are well formed.")
|
|
}
|
|
}
|
|
|
|
numberOfDownloadedReviews += len(reviewsList)
|
|
}
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
if (numberOfDownloadedReviews != 0){
|
|
numberOfReviewsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReviews)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReviewsDownloadedString + " identity reviews from host.")
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadReviewsFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadMessageReviewsFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
rangeToRetrieveStart [10]byte,
|
|
rangeToRetrieveEnd [10]byte,
|
|
reviewedMessageHashesToRetrieveList [][26]byte,
|
|
reviewersToRetrieveList [][16]byte,
|
|
reviewedMessageInboxesMap map[[26]byte][10]byte, // Message hash -> Message inbox
|
|
checkIfReviewShouldBeDownloaded func([29]byte, [26]byte)(bool, error),
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadMessageReviewsFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
|
|
//TODO: Verify inputs
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -error
|
|
downloadReviewsFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
// Outputs:
|
|
// -bool: Successfully downloaded from host
|
|
// -[21]byte: Connection identifier
|
|
// -map[[29]byte][26]byte: Map of review info: ReviewHash -> ReviewedHash
|
|
// -[10]byte: Expected Inbox Start Range of retrieved reviews
|
|
// -[10]byte: Expected Inbox End range of retrieved reviews
|
|
// -error
|
|
getRetrievedReviewsInfoMap := func()(bool, [21]byte, map[[29]byte][26]byte, [10]byte, [10]byte, error){
|
|
|
|
hostIsHostingMessages, theirInboxRangeStart, theirInboxRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (hostIsHostingMessages == false){
|
|
// Host is not hosting messages. Nothing to download.
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessagesQuantity")
|
|
}
|
|
|
|
hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity)
|
|
}
|
|
|
|
exists, hostedMessageReviewsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessageReviewsQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessageReviewsQuantity")
|
|
}
|
|
|
|
hostedMessageReviewsQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessageReviewsQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessageReviewsQuantity: " + hostedMessageReviewsQuantity)
|
|
}
|
|
|
|
getMaximumMessageHashesPerRequest := func()int{
|
|
|
|
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetMessageReviewsInfo
|
|
|
|
if (len(reviewersToRetrieveList) != 0){
|
|
|
|
// We assume each reviewer has 1 review per message
|
|
numberOfReviewersToRetrieve := len(reviewersToRetrieveList)
|
|
|
|
maximumMessageHashesPerRequest := maximumReviewsInResponse / numberOfReviewersToRetrieve
|
|
|
|
return maximumMessageHashesPerRequest
|
|
}
|
|
|
|
// We get the estimated number of reviews per message
|
|
|
|
estimatedReviewsPerMessage := hostedMessageReviewsQuantityInt64 / hostedMessagesQuantityInt64
|
|
|
|
maximumMessageHashesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerMessage
|
|
|
|
return int(maximumMessageHashesPerRequest)
|
|
}
|
|
|
|
maximumMessageHashesPerRequest := getMaximumMessageHashesPerRequest()
|
|
|
|
// We will retrieve reviews from either the reviewed hashes list, or request range
|
|
|
|
if (len(reviewedMessageHashesToRetrieveList) != 0){
|
|
|
|
// We will retrieve reviewed hashes list
|
|
|
|
messageHashesInTheirRangeList := make([][26]byte, 0)
|
|
|
|
for _, messageHash := range reviewedMessageHashesToRetrieveList{
|
|
|
|
messageInbox, exists := reviewedMessageInboxesMap[messageHash]
|
|
if (exists == false) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("reviewedMessageInboxesMap missing message hash.")
|
|
}
|
|
|
|
inboxIsWithinTheirRange, err := byteRange.CheckIfInboxIsWithinRange(theirInboxRangeStart, theirInboxRangeEnd, messageInbox)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (inboxIsWithinTheirRange == true){
|
|
messageHashesInTheirRangeList = append(messageHashesInTheirRangeList, messageHash)
|
|
}
|
|
}
|
|
|
|
sublistsToQueryList, err := helpers.SplitListIntoSublists(messageHashesInTheirRangeList, maximumMessageHashesPerRequest)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds()
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -map[[29]byte][26]byte: Reviews info map (Review hash -> Reviewed message hash)
|
|
// -error
|
|
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][26]byte, error){
|
|
|
|
retrievedReviewsInfoMap := make(map[[29]byte][26]byte)
|
|
|
|
for _, messageHashesSublist := range sublistsToQueryList{
|
|
|
|
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetMessageReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, messageHashesSublist, reviewersToRetrieveList)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, stop retrieving from this host
|
|
err := logger.AddLogEntry("Network", "Failed to download message reviews info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
|
|
retrievedReviewsInfoMap[reviewHash] = reviewedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReviewsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
|
|
if (err != nil){
|
|
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReviewsInfoMap, minimumRange, maximumRange, nil
|
|
}
|
|
|
|
// We will retrieve reviews based on message inbox range
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxRangeStart, theirInboxRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (anyIntersectionFound == false){
|
|
// Host is not hosting any reviews within the inboxes range that we are requesting
|
|
// Skip this host
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxRangeStart, theirInboxRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, int64(maximumMessageHashesPerRequest))
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -map[[29]byte][26]byte: Downloaded reviews info map (Review Hash -> Reviewed Message Hash)
|
|
// -error
|
|
getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][26]byte, error){
|
|
|
|
retrievedReviewsInfoMap := make(map[[29]byte][26]byte)
|
|
|
|
for _, subrangeObject := range subrangesToQueryList{
|
|
|
|
subrangeStart := subrangeObject.SubrangeStart
|
|
subrangeEnd := subrangeObject.SubrangeEnd
|
|
|
|
emptyList := make([][26]byte, 0)
|
|
|
|
downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetMessageReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, emptyList, reviewersToRetrieveList)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, we are done with this host
|
|
err := logger.AddLogEntry("Network", "Failed to download message reviews info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{
|
|
retrievedReviewsInfoMap[reviewHash] = reviewedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReviewsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReviewsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil
|
|
}
|
|
|
|
hostDownloadSuccessful, connectionIdentifier, retrievedReviewsInfoMap, expectedRangeStart, expectedRangeEnd, err := getRetrievedReviewsInfoMap()
|
|
if (err != nil) { return false, err }
|
|
if (hostDownloadSuccessful == false){
|
|
// We did not download any reviews from this host. Skip them.
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
// Now we see if we have any of these reviews, and if they do not fulfill the requests parameters
|
|
// If they do not fulfill the request, we know host is malicious
|
|
// This requires checking against content stored in our database
|
|
// A host could use this to learn information about what messages we are storing
|
|
|
|
// We still download the reviews to prevent host from knowing that we have the reviewed messages downloaded
|
|
// Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response
|
|
// Only after the full request is finished do we mark the host as malicious
|
|
|
|
// TODO: We shouldn't have to wait until the end of the request if we are only in Host mode
|
|
// This is because if we we don't need to worry about leaking what messages we have stored
|
|
|
|
reviewHashesToDownloadList := make([][29]byte, 0)
|
|
|
|
hostIsMalicious := false
|
|
|
|
for receivedReviewHash, receivedReviewedHash := range retrievedReviewsInfoMap{
|
|
|
|
storedReviewExists, storedReviewBytes, err := badgerDatabase.GetReview(receivedReviewHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
checkIfHostIsOfferingInvalidReview := func()(bool, error){
|
|
|
|
if (storedReviewExists == true){
|
|
|
|
ableToRead, storedReviewHash, _, reviewNetworkType, storedReviewReviewerIdentity, _, storedReviewType, storedReviewReviewedHash, _, _, err := readReviews.ReadReviewAndHash(false, storedReviewBytes)
|
|
if (err != nil) { return false, err }
|
|
if (ableToRead == false){
|
|
return false, errors.New("Database corrupt: Contains invalid review.")
|
|
}
|
|
|
|
if (storedReviewHash != receivedReviewHash){
|
|
return false, errors.New("Database corrupt: Contains review with invalid entry key hash")
|
|
}
|
|
if (reviewNetworkType != networkType){
|
|
return true, nil
|
|
}
|
|
|
|
if (len(reviewersToRetrieveList) != 0){
|
|
|
|
isInList := slices.Contains(reviewersToRetrieveList, storedReviewReviewerIdentity)
|
|
if (isInList == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
areEqual := bytes.Equal(storedReviewReviewedHash, receivedReviewedHash[:])
|
|
if (areEqual == false){
|
|
return true, nil
|
|
}
|
|
|
|
if (storedReviewType != "Message"){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
// Now we check to see if we are aware of the reviewed message's metadata, and if it matches our request parameters
|
|
|
|
metadataIsKnown, _, messageNetworkType, _, reviewedMessageInbox, _, err := contentMetadata.GetMessageMetadata(receivedReviewedHash)
|
|
if (err != nil) { return false, err }
|
|
if (metadataIsKnown == false){
|
|
return false, nil
|
|
}
|
|
if (messageNetworkType != networkType){
|
|
// The review is reviewing a message from a different networkType
|
|
// The host should have checked for this before serving us the review
|
|
// Thus, the host must be malicious
|
|
return true, nil
|
|
}
|
|
|
|
messageInboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(expectedRangeStart, expectedRangeEnd, reviewedMessageInbox)
|
|
if (err != nil) { return false, err }
|
|
if (messageInboxIsWithinRange == false){
|
|
return false, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
hostIsOfferingInvalidReview, err := checkIfHostIsOfferingInvalidReview()
|
|
if (err != nil) { return false, err }
|
|
if (hostIsOfferingInvalidReview == true){
|
|
// Host is malicious, we must still download the review
|
|
hostIsMalicious = true
|
|
|
|
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
|
|
continue
|
|
}
|
|
|
|
if (storedReviewExists == true){
|
|
|
|
// We already have the review, skip downloading it
|
|
continue
|
|
}
|
|
|
|
// Review does not exist, we must check if we want to download it
|
|
|
|
shouldDownload, err := checkIfReviewShouldBeDownloaded(receivedReviewHash, receivedReviewedHash)
|
|
if (err != nil) { return false, err }
|
|
if (shouldDownload == true){
|
|
reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash)
|
|
continue
|
|
}
|
|
}
|
|
|
|
if (len(reviewHashesToDownloadList) == 0){
|
|
// Nothing to download, done with this host.
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// Now we download reviews
|
|
|
|
// We convert the [26]byte values to []byte values to pass to GetReviewsFromHost
|
|
reviewsInfoMapForRequest := make(map[[29]byte][]byte)
|
|
|
|
for reviewHash, reviewedMessageHash := range retrievedReviewsInfoMap{
|
|
reviewsInfoMapForRequest[reviewHash] = reviewedMessageHash[:]
|
|
}
|
|
|
|
maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetReviews
|
|
|
|
reviewHashesSublistsList, err := helpers.SplitListIntoSublists(reviewHashesToDownloadList, maximumReviewsInResponse)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfDownloadedReviews := 0
|
|
|
|
for _, reviewHashesSublist := range reviewHashesSublistsList{
|
|
|
|
downloadSuccessful, reviewsList, err := sendRequests.GetReviewsFromHost(connectionIdentifier, hostIdentityHash, networkType, reviewHashesSublist, reviewsInfoMapForRequest, reviewersToRetrieveList)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Request has failed
|
|
// We will stop downloading from this host
|
|
break
|
|
}
|
|
if (hostIsMalicious == true){
|
|
// We don't add reviews from malicious hosts
|
|
continue
|
|
}
|
|
|
|
for _, reviewBytes := range reviewsList{
|
|
|
|
reviewIsWellFormed, err := reviewStorage.AddReview(reviewBytes)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (reviewIsWellFormed == false){
|
|
return false, errors.New("GetReviewsFromHost not verifying reviews are well formed.")
|
|
}
|
|
}
|
|
|
|
numberOfDownloadedReviews += len(reviewsList)
|
|
}
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
if (numberOfDownloadedReviews != 0){
|
|
numberOfReviewsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReviews)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReviewsDownloadedString + " reviews from host.")
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadReviewsFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadIdentityReportsFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
identityTypeToRetrieve string,
|
|
rangeToRetrieveStart [16]byte,
|
|
rangeToRetrieveEnd [16]byte,
|
|
reportedIdentityHashesToRetrieveList [][16]byte,
|
|
checkIfReportShouldBeDownloaded func([30]byte, []byte)(bool, error),
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadIdentityReportsFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
//TODO: Verify inputs
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -error
|
|
downloadReportsFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
// Outputs:
|
|
// -bool: Successfully downloaded from host
|
|
// -[21]byte: Connection identifier
|
|
// -map[[30]byte][]byte: Map of report info: ReportHash -> ReportedHash
|
|
// -[16]byte: Expected Identity Start Range of retrieved reports
|
|
// -[16]byte: Expected Identity End range of retrieved reports
|
|
// -[][16]byte: Expected retrieved identity hashes list
|
|
// -error
|
|
getRetrievedReportsInfoMap := func()(bool, [21]byte, map[[30]byte][]byte, [16]byte, [16]byte, [][16]byte, error){
|
|
|
|
hostIsHostingIdentityType, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, identityTypeToRetrieve)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostIsHostingIdentityType == false){
|
|
// Host is not hosting any profiles/reports of this identityType. Skip to next host
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
identitiesQuantityAttributeName := identityTypeToRetrieve + "IdentitiesQuantity"
|
|
|
|
exists, identityTypeIdentitiesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, identitiesQuantityAttributeName)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + identitiesQuantityAttributeName)
|
|
}
|
|
|
|
identityTypeIdentitiesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeIdentitiesQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains Host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity)
|
|
}
|
|
|
|
reportsQuantityAttributeName := identityTypeToRetrieve + "ReportsQuantity"
|
|
|
|
exists, identityTypeReportsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, reportsQuantityAttributeName)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + reportsQuantityAttributeName)
|
|
}
|
|
|
|
identityTypeReportsQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeReportsQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid " + reportsQuantityAttributeName + ": " + identityTypeReportsQuantity)
|
|
}
|
|
|
|
//TODO: Deal with divide by 0 case here and on all other functions
|
|
|
|
estimatedReportsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReportsQuantityInt64
|
|
|
|
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetIdentityReportsInfo
|
|
|
|
maximumIdentitiesPerRequest := int64(maximumReportsInResponse) / estimatedReportsPerIdentity
|
|
|
|
// Our request will either be based on identity hash range or identity hashes list
|
|
// We will split up the request into either ranges or identity hash lists
|
|
|
|
if (len(reportedIdentityHashesToRetrieveList) != 0){
|
|
|
|
// We will split the request into identity hash lists
|
|
|
|
hostingAny, identitiesInHostRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, reportedIdentityHashesToRetrieveList)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostingAny == false){
|
|
// Host is not hosting any of our desired identities. Skip host.
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
// Below step is just being overly cautious.
|
|
// The GetMessageReportsfromHosts function should request either a list of identities or a range. Not both.
|
|
hostingAny, identitiesInRequestRange, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identitiesInHostRangeList)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostingAny == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Requested report identity hashes list outside of request range")
|
|
}
|
|
|
|
sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInRequestRange, int(maximumIdentitiesPerRequest))
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds()
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -map[[30]byte][]byte: Downloaded reports info map (Report Hash -> Reported Hash)
|
|
// -error
|
|
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][]byte, error){
|
|
|
|
retrievedReportsInfoMap := make(map[[30]byte][]byte)
|
|
|
|
for _, identityHashesSublist := range sublistsToQueryList{
|
|
|
|
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetIdentityReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, stop downloading from this host
|
|
err := logger.AddLogEntry("Network", "Failed to download identity reports info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
|
|
retrievedReportsInfoMap[reportHash] = reportedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReportsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReportsInfoMap, minimumRange, maximumRange, identitiesInRequestRange, nil
|
|
}
|
|
|
|
// We will retrieve based on identity range, not identities list
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (anyIntersectionFound == false){
|
|
// Host is not hosting any reports within the identites range that we are requesting
|
|
// Skip this host
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
estimatedNumIdentitiesInIntersectionRange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, identityTypeIdentitiesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumIdentitiesInIntersectionRange, maximumIdentitiesPerRequest)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][]byte, error){
|
|
|
|
retrievedReportsInfoMap := make(map[[30]byte][]byte)
|
|
|
|
for _, subrangeObject := range subrangesToQueryList{
|
|
|
|
subrangeStart := subrangeObject.SubrangeStart
|
|
subrangeEnd := subrangeObject.SubrangeEnd
|
|
|
|
emptyList := make([][16]byte, 0)
|
|
|
|
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetIdentityReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, subrangeStart, subrangeEnd, emptyList)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, close connection to host.
|
|
err := logger.AddLogEntry("Network", "Failed to download identity reports info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
|
|
retrievedReportsInfoMap[reportHash] = reportedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReportsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err }
|
|
|
|
return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReportsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil, nil
|
|
}
|
|
|
|
hostDownloadSuccessful, connectionIdentifier, retrievedReportsInfoMap, expectedRangeStart, expectedRangeEnd, expectedReportedIdentityHashesList, err := getRetrievedReportsInfoMap()
|
|
if (err != nil) { return false, err }
|
|
if (hostDownloadSuccessful == false){
|
|
// We did not download any reports from this host. Skip them.
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
// Now we see if we have any of these reports, and if they do not fulfill the requests parameters
|
|
// If they do not fulfill the request, we know host is malicious
|
|
// We still download the reports to prevent host from knowing that we have those reports downloaded
|
|
// We then mark the host as malicious
|
|
|
|
reportHashesToDownloadList := make([][30]byte, 0)
|
|
|
|
hostIsMalicious := false
|
|
|
|
for receivedReportHash, receivedReportedHash := range retrievedReportsInfoMap{
|
|
|
|
//TODO: Ability to disable this step if only 1 of Host/Moderator mode is enabled
|
|
|
|
storedReportExists, storedReportBytes, err := badgerDatabase.GetReport(receivedReportHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
checkIfHostIsOfferingInvalidReport := func()(bool, error){
|
|
|
|
if (storedReportExists == true){
|
|
|
|
ableToRead, storedReportHash, _, reportNetworkType, _, storedReportType, storedReportReportedHash, _, err := readReports.ReadReportAndHash(false, storedReportBytes)
|
|
if (err != nil) { return false, err }
|
|
if (ableToRead == false){
|
|
return false, errors.New("Database corrupt: Contains invalid report")
|
|
}
|
|
if (storedReportHash != receivedReportHash){
|
|
return false, errors.New("Database corrupt: Contains report with invalid entry key hash")
|
|
}
|
|
if (reportNetworkType != networkType){
|
|
return true, nil
|
|
}
|
|
|
|
areEqual := bytes.Equal(storedReportReportedHash, receivedReportedHash)
|
|
if (areEqual == false){
|
|
return true, nil
|
|
}
|
|
|
|
if (storedReportType != "Identity" && storedReportType != "Profile" && storedReportType != "Attribute"){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
// Now we see if we have the reviewed profile metadata
|
|
|
|
reportedType, err := helpers.GetReportedTypeFromReportedHash(receivedReportedHash)
|
|
if (err != nil){ return false, err }
|
|
|
|
if (reportedType == "Profile"){
|
|
|
|
if (len(receivedReportedHash) != 28){
|
|
reportedHashHex := encoding.EncodeBytesToHexString(receivedReportedHash)
|
|
return false, errors.New("GetReportedTypeFromReportedHash returning Profile for different length reportedHash: " + reportedHashHex)
|
|
}
|
|
|
|
receivedReportedProfileHash := [28]byte(receivedReportedHash)
|
|
|
|
metadataExists, _, profileNetworkType, profileAuthor, _, profileIsDisabled, _, _, err := contentMetadata.GetProfileMetadata(receivedReportedProfileHash)
|
|
if (err != nil) { return false, err }
|
|
if (metadataExists == false){
|
|
return false, nil
|
|
}
|
|
if (profileNetworkType != networkType){
|
|
// Reported profile belongs to a different networkType than the report
|
|
// The host should have checked for this before serving this report
|
|
// Host must be malicious
|
|
return true, nil
|
|
}
|
|
if (profileIsDisabled == true){
|
|
// Disabled profiles cannot be reported
|
|
// Host must be malicious
|
|
return true, nil
|
|
}
|
|
|
|
|
|
if (expectedReportedIdentityHashesList != nil){
|
|
|
|
identityIsInList := slices.Contains(expectedReportedIdentityHashesList, profileAuthor)
|
|
if (identityIsInList == false){
|
|
return true, nil
|
|
}
|
|
|
|
} else {
|
|
|
|
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, profileAuthor)
|
|
if (err != nil) { return false, err }
|
|
if (identityIsWithinRange == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
} else if (reportedType == "Attribute"){
|
|
|
|
if (len(receivedReportedHash) != 27){
|
|
reportedHashHex := encoding.EncodeBytesToHexString(receivedReportedHash)
|
|
return false, errors.New("GetReportedTypeFromReportedHash returning Attribute for different length reportedHash: " + reportedHashHex)
|
|
}
|
|
|
|
receivedReportedAttributeHash := [27]byte(receivedReportedHash)
|
|
|
|
metadataExists, _, attributeAuthorIdentityHash, attributeNetworkType, _, err := profileStorage.GetProfileAttributeMetadata(receivedReportedAttributeHash)
|
|
if (err != nil){ return false, err }
|
|
if (metadataExists == true){
|
|
|
|
if (expectedReportedIdentityHashesList != nil){
|
|
|
|
identityIsInList := slices.Contains(expectedReportedIdentityHashesList, attributeAuthorIdentityHash)
|
|
if (identityIsInList == false){
|
|
return true, nil
|
|
}
|
|
} else {
|
|
|
|
identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, attributeAuthorIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
if (identityIsWithinRange == false){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
if (attributeNetworkType != networkType){
|
|
// Report is reporting attribute from different networkType
|
|
// Host should have checked this already. Host must be malicious.
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
// Host is not offering invalid report (as far as we know)
|
|
|
|
return false, nil
|
|
}
|
|
|
|
hostIsOfferingInvalidReport, err := checkIfHostIsOfferingInvalidReport()
|
|
if (err != nil) { return false, err }
|
|
|
|
if (hostIsOfferingInvalidReport == true){
|
|
// Host is malicious, we must still download the report
|
|
hostIsMalicious = true
|
|
|
|
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
|
|
continue
|
|
}
|
|
|
|
if (storedReportExists == true){
|
|
|
|
// We already have the report, skip downloading it
|
|
continue
|
|
}
|
|
|
|
// Report does not exist, we must check if we want to download it
|
|
// This works differently for Hosting and Moderation
|
|
|
|
shouldDownload, err := checkIfReportShouldBeDownloaded(receivedReportHash, receivedReportedHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (shouldDownload == true){
|
|
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
|
|
continue
|
|
}
|
|
}
|
|
|
|
if (len(reportHashesToDownloadList) == 0){
|
|
// Nothing to download, done with this host.
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// Now we download reports
|
|
|
|
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetReports
|
|
|
|
reportHashesSublistsList, err := helpers.SplitListIntoSublists(reportHashesToDownloadList, maximumReportsInResponse)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfDownloadedReports := 0
|
|
|
|
for _, reportHashesSublist := range reportHashesSublistsList{
|
|
|
|
downloadSuccessful, reportsList, err := sendRequests.GetReportsFromHost(connectionIdentifier, hostIdentityHash, networkType, reportHashesSublist, retrievedReportsInfoMap)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Request has failed
|
|
// We will stop downloading from this host
|
|
break
|
|
}
|
|
|
|
for _, reportBytes := range reportsList{
|
|
|
|
reportIsWellFormed, err := reportStorage.AddReport(reportBytes)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (reportIsWellFormed == false){
|
|
return false, errors.New("GetReportsFromHost not verifying reports are well formed.")
|
|
}
|
|
}
|
|
|
|
numberOfDownloadedReports += len(reportsList)
|
|
}
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
if (numberOfDownloadedReports != 0){
|
|
numberOfReportsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReports)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReportsDownloadedString + " identity reports from host.")
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadReportsFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadMessageReportsFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
rangeToRetrieveStart [10]byte,
|
|
rangeToRetrieveEnd [10]byte,
|
|
reportedMessageHashesToRetrieveList [][26]byte,
|
|
reportedMessagesInboxMap map[[26]byte][10]byte, // Message hash -> Message inbox
|
|
checkIfReportShouldBeDownloaded func([30]byte, [26]byte)(bool, error),
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadMessageReportsFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
//TODO: Verify inputs
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -error
|
|
downloadReportsFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
// Outputs:
|
|
// -bool: Successfully downloaded from host
|
|
// -[21]byte: Connection identifier
|
|
// -map[[30]byte][26]byte: Map of report info: ReportHash-> Reported message Hash
|
|
// -[10]byte: Expected Inbox Start Range of retrieved reports
|
|
// -[10]byte: Expected Inbox End range of retrieved reports
|
|
// -error
|
|
getRetrievedReportsInfoMap := func()(bool, [21]byte, map[[30]byte][26]byte, [10]byte, [10]byte, error){
|
|
|
|
hostIsHostingMessages, theirInboxHostRangeStart, theirInboxHostRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (hostIsHostingMessages == false){
|
|
|
|
// Host is not hosting messages. Nothing to download.
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessagesQuantity")
|
|
}
|
|
|
|
hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity)
|
|
}
|
|
|
|
exists, hostedMessageReportsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessageReportsQuantity")
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (exists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessageReportsQuantity")
|
|
}
|
|
|
|
hostedMessageReportsQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessageReportsQuantity)
|
|
if (err != nil) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Databse corrupt: Contains host profile with invalid MessageReportsQuantity: " + hostedMessageReportsQuantity)
|
|
}
|
|
|
|
if (hostedMessageReportsQuantityInt64 == 0){
|
|
// Host is not hosting any message reports. Nothing to download.
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetMessageReportsInfo
|
|
|
|
estimatedReportsPerMessage := hostedMessageReportsQuantityInt64 / hostedMessagesQuantityInt64
|
|
|
|
maximumMessageHashesPerRequest := int64(maximumReportsInResponse) / estimatedReportsPerMessage
|
|
|
|
// We will retrieve reports from either the reported hashes list, or request range
|
|
|
|
if (len(reportedMessageHashesToRetrieveList) != 0){
|
|
|
|
// We will retrieve reported message hashes list
|
|
|
|
messageHashesInTheirRangeList := make([][26]byte, 0)
|
|
|
|
for _, messageHash := range reportedMessageHashesToRetrieveList{
|
|
|
|
messageInbox, exists := reportedMessagesInboxMap[messageHash]
|
|
if (exists == false) {
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("ReportedMessagesInboxMap missing message hash.")
|
|
}
|
|
|
|
inboxIsWithinTheirRange, err := byteRange.CheckIfInboxIsWithinRange(theirInboxHostRangeStart, theirInboxHostRangeEnd, messageInbox)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
if (inboxIsWithinTheirRange == true){
|
|
messageHashesInTheirRangeList = append(messageHashesInTheirRangeList, messageHash)
|
|
}
|
|
}
|
|
|
|
sublistsToQueryList, err := helpers.SplitListIntoSublists(messageHashesInTheirRangeList, int(maximumMessageHashesPerRequest))
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds()
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported Message Hash)
|
|
// -error
|
|
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][26]byte, error){
|
|
|
|
retrievedReportsInfoMap := make(map[[30]byte][26]byte)
|
|
|
|
for _, messageHashesSublist := range sublistsToQueryList{
|
|
|
|
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetMessageReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, messageHashesSublist)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, end connection to host
|
|
err := logger.AddLogEntry("Network", "Failed to download message reports info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
|
|
retrievedReportsInfoMap[reportHash] = reportedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReportsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReportsInfoMap, minimumRange, maximumRange, nil
|
|
}
|
|
|
|
// We will retrieve reports based on message inbox range
|
|
|
|
anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxHostRangeStart, theirInboxHostRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (anyIntersectionFound == false){
|
|
// Host is not hosting any reports within the inboxes range that we are requesting
|
|
// Skip this host
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxHostRangeStart, theirInboxHostRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, maximumMessageHashesPerRequest)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
if (hostProfileExists == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Download successful
|
|
// -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported message hash)
|
|
// -error
|
|
getDownloadedReportsInfoMap := func()(bool, map[[30]byte][26]byte, error){
|
|
|
|
retrievedReportsInfoMap := make(map[[30]byte][26]byte)
|
|
|
|
for _, subrangeObject := range subrangesToQueryList{
|
|
|
|
subrangeStart := subrangeObject.SubrangeStart
|
|
subrangeEnd := subrangeObject.SubrangeEnd
|
|
|
|
emptyList := make([][26]byte, 0)
|
|
|
|
downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetMessageReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, emptyList)
|
|
if (err != nil) { return false, nil, err }
|
|
if (downloadSuccessful == false){
|
|
// Failed to download this sublist.
|
|
// Could be that host went offline permanently or just this connection failed.
|
|
// Either way, stop downloading from host
|
|
err := logger.AddLogEntry("Network", "Failed to download message reports info from host.")
|
|
if (err != nil) { return false, nil, err }
|
|
|
|
return false, nil, nil
|
|
}
|
|
|
|
for reportHash, reportedHash := range currentRetrievedReportsInfoMap{
|
|
retrievedReportsInfoMap[reportHash] = reportedHash
|
|
}
|
|
}
|
|
|
|
return true, retrievedReportsInfoMap, nil
|
|
}
|
|
|
|
downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap()
|
|
if (err != nil) {
|
|
errB := peerClient.CloseConnection(connectionIdentifier)
|
|
if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err
|
|
}
|
|
if (downloadSuccessful == false){
|
|
|
|
err := peerClient.CloseConnection(connectionIdentifier)
|
|
if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err }
|
|
|
|
return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil
|
|
}
|
|
|
|
return true, connectionIdentifier, retrievedReportsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil
|
|
}
|
|
|
|
hostDownloadSuccessful, connectionIdentifier, retrievedReportsInfoMap, expectedRangeStart, expectedRangeEnd, err := getRetrievedReportsInfoMap()
|
|
if (err != nil) { return false, err }
|
|
if (hostDownloadSuccessful == false){
|
|
// We did not download any reports from this host. Skip them.
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
// Now we see if we have any of these reports, and if they do not fulfill the requests parameters
|
|
// If they do not fulfill the request, we know host is malicious
|
|
// We still download the reports to prevent host from knowing that we have those reports downloaded
|
|
// We then mark the host as malicious
|
|
|
|
reportHashesToDownloadList := make([][30]byte, 0)
|
|
|
|
hostIsMalicious := false
|
|
|
|
for receivedReportHash, receivedReportedHash := range retrievedReportsInfoMap{
|
|
|
|
//TODO: Disable this step if only 1 of Host/Moderator mode is enabled
|
|
|
|
storedReportExists, storedReportBytes, err := badgerDatabase.GetReport(receivedReportHash)
|
|
if (err != nil) { return false, err }
|
|
|
|
checkIfHostIsOfferingInvalidReport := func()(bool, error){
|
|
|
|
if (storedReportExists == true){
|
|
|
|
ableToRead, storedReportHash, _, reportNetworkType, _, storedReportType, storedReportReportedHash, _, err := readReports.ReadReportAndHash(false, storedReportBytes)
|
|
if (err != nil) { return false, err }
|
|
if (ableToRead == false){
|
|
return false, errors.New("Database corrupt: Contains invalid report")
|
|
}
|
|
if (storedReportHash != receivedReportHash){
|
|
return false, errors.New("Database corrupt: Contains report with invalid entry key hash")
|
|
}
|
|
if (reportNetworkType != networkType){
|
|
return true, nil
|
|
}
|
|
|
|
areEqual := bytes.Equal(storedReportReportedHash, receivedReportedHash[:])
|
|
if (areEqual == false){
|
|
return true, nil
|
|
}
|
|
|
|
if (storedReportType != "Message"){
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
// Now we see if we have the reviewed message metadata
|
|
|
|
metadataExists, _, messageNetworkType, _, reportedMessageInbox, _, err := contentMetadata.GetMessageMetadata(receivedReportedHash)
|
|
if (err != nil) { return false, err }
|
|
if (metadataExists == false){
|
|
return false, nil
|
|
}
|
|
if (messageNetworkType != networkType){
|
|
// Report is reporting a message on a different networkType than the report
|
|
// The host should have checked for this already
|
|
// Host must be malicious
|
|
return true, nil
|
|
}
|
|
|
|
inboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(expectedRangeStart, expectedRangeEnd, reportedMessageInbox)
|
|
if (err != nil) { return false, err }
|
|
if (inboxIsWithinRange == false){
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
hostIsOfferingInvalidReport, err := checkIfHostIsOfferingInvalidReport()
|
|
if (err != nil) { return false, err }
|
|
|
|
if (hostIsOfferingInvalidReport == true){
|
|
// Host is malicious, we must still download the report
|
|
hostIsMalicious = true
|
|
|
|
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
|
|
continue
|
|
}
|
|
|
|
if (storedReportExists == true){
|
|
|
|
// We already have the report, skip downloading it
|
|
continue
|
|
}
|
|
|
|
// Report does not exist, we must check if we want to download it
|
|
// This works differently for Hosting and Moderation
|
|
// For instance, if the user is downloading reports for moderation, they will not redownload reports for messages that they have already reviewed
|
|
|
|
shouldDownload, err := checkIfReportShouldBeDownloaded(receivedReportHash, receivedReportedHash)
|
|
if (err != nil) { return false, err }
|
|
if (shouldDownload == true){
|
|
reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash)
|
|
continue
|
|
}
|
|
}
|
|
|
|
if (len(reportHashesToDownloadList) == 0){
|
|
// Nothing to download, done with this host.
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Now we download reports
|
|
|
|
// We convert the [26]byte value to []byte to supply to GetReportsFromHost
|
|
reportsInfoMapForRequest := make(map[[30]byte][]byte)
|
|
|
|
for reportHash, reportedMessageHash := range retrievedReportsInfoMap{
|
|
|
|
reportsInfoMapForRequest[reportHash] = reportedMessageHash[:]
|
|
}
|
|
|
|
maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetReports
|
|
|
|
reportHashesSublistsList, err := helpers.SplitListIntoSublists(reportHashesToDownloadList, maximumReportsInResponse)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfDownloadedReports := 0
|
|
|
|
for _, reportHashesSublist := range reportHashesSublistsList{
|
|
|
|
downloadSuccessful, reportsList, err := sendRequests.GetReportsFromHost(connectionIdentifier, hostIdentityHash, networkType, reportHashesSublist, reportsInfoMapForRequest)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// Request has failed
|
|
// We will stop downloading from this host
|
|
break
|
|
}
|
|
|
|
for _, reportBytes := range reportsList{
|
|
|
|
reportIsWellFormed, err := reportStorage.AddReport(reportBytes)
|
|
if (err != nil) { return false, err }
|
|
|
|
if (reportIsWellFormed == false){
|
|
return false, errors.New("GetReportsFromHost not verifying reports are well formed.")
|
|
}
|
|
}
|
|
|
|
numberOfDownloadedReports += len(reportsList)
|
|
}
|
|
|
|
if (hostIsMalicious == true){
|
|
err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
if (numberOfDownloadedReports != 0){
|
|
numberOfReportsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReports)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReportsDownloadedString + " reports from host.")
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadReportsFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
//Inputs:
|
|
// -bool: Allow clearnet
|
|
// -[]string: List of identity hashes to retrieve viewable consensus statuses for
|
|
// -map[[28]byte][16]byte: Map of Profile Hash -> Profile Identity Hash
|
|
// Outputs:
|
|
// -error
|
|
func DownloadViewableStatusesFromHosts(
|
|
allowClearnet bool,
|
|
networkType byte,
|
|
knownOrUnknown string,
|
|
identityHashesToRetrieveList [][16]byte,
|
|
profileHashesToRetrieveMap map[[28]byte][16]byte,
|
|
numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadViewableStatusesFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
if (knownOrUnknown != "Known" && knownOrUnknown != "Unknown"){
|
|
return errors.New("DownloadViewableStatusesFromHosts called with invalid knownOrUnknown: " + knownOrUnknown)
|
|
}
|
|
|
|
//Outputs:
|
|
// -bool: Downloaded any statuses
|
|
// -error
|
|
downloadViewableStatusesFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (exists == false){
|
|
// Host profile was deleted, skip host
|
|
return false, nil
|
|
}
|
|
|
|
hostIsHostingMateProfiles, hostMateRangeStart, hostMateRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Mate")
|
|
if (err != nil) { return false, err }
|
|
hostIsHostingHostProfiles, _, _, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Host")
|
|
if (err != nil) { return false, err }
|
|
hostIsHostingModeratorProfiles, _, _, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Moderator")
|
|
if (err != nil) { return false, err }
|
|
|
|
checkIfHostIsHostingIdentityHash := func(identityHash [16]byte)(bool, error){
|
|
|
|
identityType, err := identity.GetIdentityTypeFromIdentityHash(identityHash)
|
|
if (err != nil) { return false, err }
|
|
if (identityType == "Mate"){
|
|
|
|
if (hostIsHostingMateProfiles == false){
|
|
return false, nil
|
|
}
|
|
|
|
isWithinTheirRange, err := byteRange.CheckIfIdentityHashIsWithinRange(hostMateRangeStart, hostMateRangeEnd, identityHash)
|
|
if (err != nil) { return false, err }
|
|
if (isWithinTheirRange == false){
|
|
return false, nil
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
if (identityType == "Host"){
|
|
if (hostIsHostingHostProfiles == false){
|
|
return false, nil
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
// identityType == "Moderator"
|
|
if (hostIsHostingModeratorProfiles == false){
|
|
return false, nil
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
relevantIdentityHashesToRetrieveList := make([][16]byte, 0)
|
|
|
|
for _, identityHash := range identityHashesToRetrieveList{
|
|
|
|
isHostingIdentity, err := checkIfHostIsHostingIdentityHash(identityHash)
|
|
if (err != nil) { return false, err }
|
|
if (isHostingIdentity == false){
|
|
continue
|
|
}
|
|
|
|
statusIsKnown, _, queriedHostsList, err := trustedViewableStatus.GetTrustedIdentityIsViewableStatus(identityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (knownOrUnknown == "Known"){
|
|
// We only want to retrieve known statuses. We dont need to avoid already queried hosts.
|
|
if (statusIsKnown == false){
|
|
continue
|
|
}
|
|
} else {
|
|
// We only want to retrieve unknown statuses
|
|
if (statusIsKnown == true){
|
|
continue
|
|
}
|
|
|
|
// We must avoid hosts we have already queried
|
|
alreadyQueried := slices.Contains(queriedHostsList, hostIdentityHash)
|
|
if (alreadyQueried == true){
|
|
continue
|
|
}
|
|
}
|
|
|
|
relevantIdentityHashesToRetrieveList = append(relevantIdentityHashesToRetrieveList, identityHash)
|
|
}
|
|
|
|
relevantProfileHashesToRetrieveList := make([][28]byte, 0)
|
|
|
|
for profileHash, profileIdentityHash := range profileHashesToRetrieveMap{
|
|
|
|
isHostingIdentity, err := checkIfHostIsHostingIdentityHash(profileIdentityHash)
|
|
if (err != nil) { return false, err }
|
|
if (isHostingIdentity == false){
|
|
continue
|
|
}
|
|
|
|
statusIsKnown, _, queriedHostsList, err := trustedViewableStatus.GetTrustedProfileIsViewableStatus(profileHash)
|
|
if (err != nil) { return false, err }
|
|
if (knownOrUnknown == "Known"){
|
|
// We only want to retrieve known statuses. We dont need to avoid already queried hosts.
|
|
if (statusIsKnown == false){
|
|
continue
|
|
}
|
|
} else {
|
|
// We only want to retrieve unknown statuses
|
|
if (statusIsKnown == true){
|
|
continue
|
|
}
|
|
|
|
// We must avoid hosts we have already queried
|
|
alreadyQueried := slices.Contains(queriedHostsList, hostIdentityHash)
|
|
if (alreadyQueried == true){
|
|
continue
|
|
}
|
|
}
|
|
|
|
relevantProfileHashesToRetrieveList = append(relevantProfileHashesToRetrieveList, profileHash)
|
|
}
|
|
|
|
if (len(relevantIdentityHashesToRetrieveList) == 0 && len(relevantProfileHashesToRetrieveList) == 0){
|
|
// Nothing to retrieve from this host. Skip them.
|
|
return false, nil
|
|
}
|
|
|
|
// Now we must split up our request to not exceed maximum size
|
|
// The requests/responses for identity hashes will always be smaller than profile hashes, for the same number of identities/profiles
|
|
// This is because profile hashes are larger than identity hashes
|
|
// We will split request so that we can fit the maximum amount of identity/profile hashes in all of the requests
|
|
|
|
numberOfIdentityHashesToRetrieve := len(relevantIdentityHashesToRetrieveList)
|
|
numberOfProfileHashesToRetrieve := len(relevantProfileHashesToRetrieveList)
|
|
|
|
maximumIdentityHashesInResponse := serverResponse.MaximumIdentitiesInResponse_GetIdentityViewableStatuses
|
|
maximumProfileHashesInResponse := serverResponse.MaximumProfilesInResponse_GetProfileViewableStatuses
|
|
|
|
numberOfRequestsNeededForIdentityHashes := float64(numberOfIdentityHashesToRetrieve)/float64(maximumIdentityHashesInResponse)
|
|
numberOfRequestsNeededForProfileHashes := float64(numberOfProfileHashesToRetrieve)/float64(maximumProfileHashesInResponse)
|
|
|
|
totalNumberOfRequestsNeeded, err := helpers.CeilFloat64ToInt64(numberOfRequestsNeededForIdentityHashes + numberOfRequestsNeededForProfileHashes)
|
|
if (err != nil) { return false, err }
|
|
|
|
numberOfIdentityHashesPerRequest := int64(numberOfIdentityHashesToRetrieve)/totalNumberOfRequestsNeeded
|
|
numberOfProfileHashesPerRequest := int64(numberOfProfileHashesToRetrieve)/totalNumberOfRequestsNeeded
|
|
|
|
identityHashesSublistsList, err := helpers.SplitListIntoSublists(relevantIdentityHashesToRetrieveList, int(numberOfIdentityHashesPerRequest))
|
|
if (err != nil) { return false, err }
|
|
profileHashesSublistsList, err := helpers.SplitListIntoSublists(relevantProfileHashesToRetrieveList, int(numberOfProfileHashesPerRequest))
|
|
if (err != nil) { return false, err }
|
|
|
|
identityHashesSublistsListLength := len(identityHashesSublistsList)
|
|
profileHashesSublistsListLength := len(profileHashesSublistsList)
|
|
|
|
getLongerListLength := func()int{
|
|
if (identityHashesSublistsListLength > profileHashesSublistsListLength){
|
|
return identityHashesSublistsListLength
|
|
}
|
|
return profileHashesSublistsListLength
|
|
}
|
|
|
|
longerListLength := getLongerListLength()
|
|
|
|
totalDownloadedStatuses := 0
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, err }
|
|
if (hostProfileExists == false){
|
|
return false, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
for i := 0; i < longerListLength; i++ {
|
|
|
|
getIdentityHashesSublist := func()[][16]byte{
|
|
if (i < identityHashesSublistsListLength){
|
|
return identityHashesSublistsList[i]
|
|
}
|
|
emptyList := make([][16]byte, 0)
|
|
return emptyList
|
|
}
|
|
|
|
getProfileHashesSublist := func()[][28]byte{
|
|
if (i < profileHashesSublistsListLength){
|
|
return profileHashesSublistsList[i]
|
|
}
|
|
emptyList := make([][28]byte, 0)
|
|
return emptyList
|
|
}
|
|
|
|
identityHashesSublist := getIdentityHashesSublist()
|
|
profileHashesSublist := getProfileHashesSublist()
|
|
|
|
downloadSuccessful, identityHashStatusesMap, profileHashStatusesMap, err := sendRequests.GetViewableStatusesFromHost(connectionIdentifier, hostIdentityHash, networkType, identityHashesSublist, profileHashesSublist)
|
|
if (err != nil) { return false, err }
|
|
if (downloadSuccessful == false){
|
|
// This host has failed to respond. We will stop downloading from them.
|
|
return false, nil
|
|
}
|
|
|
|
for identityHash, viewableStatus := range identityHashStatusesMap{
|
|
|
|
err := trustedViewableStatus.AddTrustedIdentityIsViewableStatus(identityHash, hostIdentityHash, networkType, viewableStatus)
|
|
if (err != nil) { return false, err }
|
|
|
|
totalDownloadedStatuses += 1
|
|
}
|
|
|
|
for profileHash, viewableStatus := range profileHashStatusesMap{
|
|
|
|
err := trustedViewableStatus.AddTrustedProfileIsViewableStatus(profileHash, hostIdentityHash, viewableStatus)
|
|
if (err != nil) { return false, err }
|
|
|
|
profileIdentityHash, exists := profileHashesToRetrieveMap[profileHash]
|
|
if (exists == false){
|
|
return false, errors.New("profileHashesToRetrieveMap missing profileHash")
|
|
}
|
|
|
|
if (viewableStatus == true){
|
|
// The host will only indicate a viewable profile if they know that the identity is not banned
|
|
err := trustedViewableStatus.AddTrustedIdentityIsViewableStatus(profileIdentityHash, hostIdentityHash, networkType, true)
|
|
if (err != nil) { return false, err }
|
|
}
|
|
|
|
totalDownloadedStatuses += 1
|
|
}
|
|
}
|
|
|
|
totalDownloadedStatusesString := helpers.ConvertIntToString(totalDownloadedStatuses)
|
|
|
|
err = logger.AddLogEntry("Network", "Successfully downloaded " + totalDownloadedStatusesString + " viewable statuses from host.")
|
|
if (err != nil) { return false, err }
|
|
|
|
return true, nil
|
|
}
|
|
|
|
eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil) { return err }
|
|
|
|
queriedHosts := 0
|
|
|
|
for _, hostIdentityHash := range eligibleHostsList{
|
|
|
|
successfulDownload, err := downloadViewableStatusesFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (successfulDownload == true){
|
|
queriedHosts += 1
|
|
}
|
|
if (queriedHosts >= numberOfHostsToQuery){
|
|
return nil
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
func DownloadModeratorAddressDepositsFromHosts(allowClearnet bool, networkType byte, cryptocurrencyName string, knownOrUnknown string, numberOfHostsToQuery int)error{
|
|
|
|
isValid := helpers.VerifyNetworkType(networkType)
|
|
if (isValid == false){
|
|
networkTypeString := helpers.ConvertByteToString(networkType)
|
|
return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid networkType: " + networkTypeString)
|
|
}
|
|
|
|
if (cryptocurrencyName != "Ethereum" && cryptocurrencyName != "Cardano"){
|
|
return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid cryptocurrencyName: " + cryptocurrencyName)
|
|
}
|
|
|
|
if (knownOrUnknown != "Unknown" && knownOrUnknown != "Known"){
|
|
return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid knownOrUnknown: " + knownOrUnknown)
|
|
}
|
|
|
|
//TODO: Check if local Ethereum/Cardano node is being used (either here or in function that calls this function)
|
|
//TODO: Check to see if less than 3 blockchain hosts exist, in which case, relax the required rule of 3 needed
|
|
|
|
// We will first see which identity deposits are known/unknown
|
|
|
|
//Outputs:
|
|
// -bool: Downloaded any deposits
|
|
// -error
|
|
downloadDepositsFromHost := func(hostIdentityHash [16]byte)(bool, error){
|
|
|
|
profileExists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType)
|
|
if (err != nil) { return false, err }
|
|
if (profileExists == false){
|
|
return false, nil
|
|
}
|
|
|
|
exists, isHostingBlockchain, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "Hosting" + cryptocurrencyName + "Blockchain")
|
|
if (err != nil) { return false, err }
|
|
if (exists == false || isHostingBlockchain != "Yes"){
|
|
return false, nil
|
|
}
|
|
|
|
localBlockchainNodeEnabled, addressesToUpdateList, err := moderatorScores.GetModeratorCryptoDepositAddressesList(cryptocurrencyName, knownOrUnknown)
|
|
if (err != nil){ return false, err }
|
|
if (localBlockchainNodeEnabled == true){
|
|
// We must have enabled the node after our earlier check
|
|
// Skip all remaining hosts.
|
|
return false, nil
|
|
}
|
|
|
|
if (len(addressesToUpdateList) == 0){
|
|
// No addresses to update remain.
|
|
return false, nil
|
|
}
|
|
|
|
//TODO: Add limit to size of below
|
|
|
|
hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType)
|
|
if (err != nil){ return false, err }
|
|
if (hostProfileExists == false){
|
|
return false, nil
|
|
}
|
|
if (connectionEstablished == false){
|
|
return false, nil
|
|
}
|
|
|
|
defer peerClient.CloseConnection(connectionIdentifier)
|
|
|
|
successfulDownload, retrievedDepositObjectsList, addressesWithNoDepositsList, err := sendRequests.GetAddressDepositsFromHost(connectionIdentifier, hostIdentityHash, networkType, cryptocurrencyName, addressesToUpdateList)
|
|
if (err != nil) { return false, err }
|
|
if (successfulDownload == false){
|
|
err := logger.AddLogEntry("Network", "Failed to retrieve address deposits from host.")
|
|
if (err != nil) { return false, err }
|
|
// Skip to next host
|
|
return false, nil
|
|
}
|
|
|
|
err = trustedAddressDeposits.AddAddressDepositObjectsListToCache(hostIdentityHash, cryptocurrencyName, retrievedDepositObjectsList)
|
|
if (err != nil) { return false, err }
|
|
|
|
err = trustedAddressDeposits.AddAddressesWithNoDepositsToCache(hostIdentityHash, cryptocurrencyName, addressesWithNoDepositsList)
|
|
if (err != nil) { return false, err }
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// We query each host for the identity deposits we want
|
|
// After each host, we get the new known/unknown identities list to make
|
|
// sure we do not query deposits which were unknown but have become known
|
|
|
|
hostsToQueryList, err := eligibleHosts.GetEligibleHostsList(networkType)
|
|
if (err != nil){ return err }
|
|
|
|
hostsQueried := 0
|
|
for _, hostIdentityHash := range hostsToQueryList{
|
|
|
|
downloadedAnyDeposits, err := downloadDepositsFromHost(hostIdentityHash)
|
|
if (err != nil) { return err }
|
|
if (downloadedAnyDeposits == true){
|
|
hostsQueried += 1
|
|
}
|
|
|
|
if (hostsQueried >= numberOfHostsToQuery){
|
|
break
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
|
|
|
|
|