// queryHosts provides functions to download information from hosts // These include parameters, profiles, messages, reports, reviews, identity deposits, and trusted moderation viewable statuses // Each function will query a specified number of eligible hosts package queryHosts //TODO: If you download a user's identity, then the host is saying that the user's identity is funded. Add it to trustedFundedStatus //TODO: Split requests up so that memory does not have to store all of the requested hashes //TODO: Set a maximum limit, so that only a certain amount of data will be requested from each host //TODO: Choose hosts more intelligently based on what ranges we are requesting // Right now, we are sending requests to randomly chosen hosts until we reach the numberOfHostsToQuery // This is a bad strategy, because, if we are requesting range 20-100, we might request from a host that is only hosting from range 10-25 // We should use numberOfHostsToQuery to choose a set of hosts that will host the entire range // We must make sure that we sometimes request from hosts that only host a small number of total profiles. // TODO: Add BroadcastToHosts, GetFundedStatuses, and more //TODO: If only Host/Moderator mode is enabled, we can skip all of the downloaded identity/inbox fingerprinting prevention logic // This logic is designed to prevent hosts from learning the user's moderator/host identity/behavior from their downloaded content // For example, a user is moderating and hosting. requestors could determine a host's moderator identity by querying for // profiles outside of their host range and seeing which mate profiles they have downloaded. // Using this information, they could learn their moderator range and link it to their moderator identity/profile. // // We cannot disable fingerprinting if Mate mode is enabled, because a user's Mate criteria will change, so we cannot allow the host to know what their criteria was previously // This is also true if a Host or Moderator's ranges change, but in those cases, the leak of privacy is fine // This is because the information (previous host/moderator ranges) is not sensitive, and not a problem if it is leaked import "seekia/internal/byteRange" import "seekia/internal/badgerDatabase" import "seekia/internal/contentMetadata" import "seekia/internal/encoding" import "seekia/internal/helpers" import "seekia/internal/identity" import "seekia/internal/logger" import "seekia/internal/messaging/chatMessageStorage" import "seekia/internal/moderation/moderatorScores" import "seekia/internal/moderation/readReports" import "seekia/internal/moderation/readReviews" import "seekia/internal/moderation/reportStorage" import "seekia/internal/moderation/reviewStorage" import "seekia/internal/moderation/trustedAddressDeposits" import "seekia/internal/moderation/trustedViewableStatus" import "seekia/internal/network/eligibleHosts" import "seekia/internal/network/hostRanges" import "seekia/internal/network/maliciousHosts" import "seekia/internal/network/mateCriteria" import "seekia/internal/network/peerClient" import "seekia/internal/network/sendRequests" import "seekia/internal/network/serverResponse" import "seekia/internal/parameters/parametersStorage" import "seekia/internal/profiles/profileStorage" import "seekia/internal/profiles/readProfiles" import "bytes" import "errors" import "slices" func DownloadParametersFromHosts(allowClearnet bool, networkType byte, numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadParametersFromHosts called with invalid networkType: " + networkTypeString) } // Outputs: // -bool: Successful download from host // -error downloadParametersFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } exists, hostIsHostingParameters, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "HostingParameters") if (err != nil) { return false, err } if (exists == false){ return false, errors.New("Database corrupt: Contains host Profile missing HostingParameters") } if (hostIsHostingParameters != "Yes"){ // This host is not hosting parameters, skip return false, nil } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, err } if (hostProfileExists == false){ return false, nil } if (connectionEstablished == false){ return false, nil } defer peerClient.CloseConnection(connectionIdentifier) downloadSuccessful, parametersInfoMap, err := sendRequests.GetParametersInfoFromHost(connectionIdentifier, hostIdentityHash, networkType) if (err != nil) { return false, err } if (downloadSuccessful == false){ return false, nil } if (len(parametersInfoMap) == 0){ // The host has provided no parameters. We will not consider this a successful download return false, nil } // We will check to see if our parameters creation time is older than the one the host is offering // If it is, we will download it and continue parametersTypesToRetrieveList := make([]string, 0) for parametersType, parametersCreationTime := range parametersInfoMap{ storedParametersFound, _, _, storedParametersCreationTime, err := parametersStorage.GetAuthorizedParameters(parametersType, networkType) if (err != nil) { return false, err } if (storedParametersFound == true){ if (storedParametersCreationTime < parametersCreationTime){ // The offered parameters are not newer than our existing parameters // Skip downloading them. continue } } parametersTypesToRetrieveList = append(parametersTypesToRetrieveList, parametersType) } if (len(parametersTypesToRetrieveList) == 0){ // The host has no parameters that we need. Nothing to download. return true, nil } // Now we download parameters downloadSuccessful, receivedParametersList, err := sendRequests.GetParametersFromHost(connectionIdentifier, hostIdentityHash, networkType, parametersTypesToRetrieveList) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Unable to download parameters. Nothing left to do with this host. return false, nil } if (len(receivedParametersList) == 0){ // Host is probably malicious // Host could have received new admin permissions after we got parameters info // Either way, we do not count this as a successful download return false, nil } for _, parametersBytes := range receivedParametersList{ parametersAreValid, _, err := parametersStorage.AddParameters(parametersBytes) if (err != nil) { return false, err } if (parametersAreValid == false){ return false, errors.New("GetParametersFromHost not verifying parameters.") } } // These parameters were downloaded, not necessarily authorized or added numberOfDownloadedParametersString := helpers.ConvertIntToString(len(receivedParametersList)) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfDownloadedParametersString + " network parameters from host.") if (err != nil) { return false, err } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadParametersFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadProfilesFromHosts( allowClearnet bool, networkType byte, profileTypeToRetrieve string, rangeToRetrieveStart [16]byte, rangeToRetrieveEnd [16]byte, identityHashesToRetrieveList [][16]byte, criteria []byte, getNewestProfilesOnly bool, getViewableProfilesOnly bool, downloadAllMode bool, checkIfProfileShouldBeDownloaded func([28]byte, [16]byte, int64)(bool, error), numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadProfilesFromHosts called with invalid networkType: " + networkTypeString) } // We cycle through each host // We get the profiles info for profiles that are within our requested criteria/range/identitites list // We see if any are knowingly outside of criteria, and download anyway if they are (to resist fingerprinting) // We download the profiles. // We don't download any profiles that we already have (if desiresPruningMode == false) //TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable profiles //Outputs: // -bool: Successfully downloaded from host (false if host does not have any profiles we need) // -error downloadProfilesFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } hostIsHostingProfileType, theirHostRangeStart, theirHostRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, profileTypeToRetrieve) if (err != nil) { return false, err } if (hostIsHostingProfileType == false){ // Host is not hosting any profiles of this profileType. Skip to next host return false, nil } //Outputs: // -bool: Download successful // -[21]byte: Connection identifier // -[]serverResponse.ProfileInfoStruct: Retrieved profiles info objects list // -error getRetrievedProfileInfoObjectsList := func()(bool, [21]byte, []serverResponse.ProfileInfoStruct, error){ // Our request will either be based on identity hash range or identity hash list // We will split up the request into either ranges or identity hashes maximumProfilesInGetProfilesInfoRequest := serverResponse.MaximumProfilesInResponse_GetProfilesInfo - 100 if (len(identityHashesToRetrieveList) != 0){ // We will retrieve identities lists, not identity ranges anyValuesExist, identitiesInMyRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identityHashesToRetrieveList) if (err != nil) { return false, [21]byte{}, nil, err } if (anyValuesExist == false){ // This should not happen return false, [21]byte{}, nil, errors.New("Identity hashes list to retrieve contains no identities within request range.") } anyValuesExist, identitiesInTheirRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostRangeStart, theirHostRangeEnd, identitiesInMyRangeList) if (err != nil) { return false, [21]byte{}, nil, err } if (anyValuesExist == false){ // This host is not hosting any identities we desire. We will skip them. return false, [21]byte{}, nil, nil } // Now we split request into sublists if needed getMaximumIdentitiesPerRequest := func()int{ if (getNewestProfilesOnly == true){ return maximumProfilesInGetProfilesInfoRequest } // We assume each identity has approximately 3 profiles maximumIdentitiesPerRequest := maximumProfilesInGetProfilesInfoRequest / 3 return maximumIdentitiesPerRequest } maximumIdentitiesPerRequest := getMaximumIdentitiesPerRequest() sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInTheirRangeList, maximumIdentitiesPerRequest) if (err != nil) { return false, [21]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, nil } downloadProfileInfoObjectsList := func()(bool, []serverResponse.ProfileInfoStruct, error){ retrievedProfileInfoObjectsList := make([]serverResponse.ProfileInfoStruct, 0) minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds() for _, identityHashesSublist := range sublistsToQueryList{ downloadSuccessful, profilesInfoObjectsList, err := sendRequests.GetProfilesInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist, criteria, getNewestProfilesOnly, getViewableProfilesOnly) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this subrange. // Could be that host went offline permanently or just this connection failed. // Either way, cease connection to host err := logger.AddLogEntry("Network", "Failed to download profiles info from host.") if (err != nil) { return false, nil, err } err = peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, nil, err } return false, nil, nil } retrievedProfileInfoObjectsList = append(retrievedProfileInfoObjectsList, profilesInfoObjectsList...) } return true, retrievedProfileInfoObjectsList, nil } downloadSuccessful, retrievedProfileInfoObjectsList, err := downloadProfileInfoObjectsList() if (err != nil){ errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, errB } return false, [21]byte{}, nil, err } if (downloadSuccessful == false){ err = peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, err } return false, [21]byte{}, nil, nil } return true, connectionIdentifier, retrievedProfileInfoObjectsList, nil } // We will retrieve identity ranges, not identity lists //Outputs: // -Any intersection found // -[16]byte: Intersection range start // -[16]byte: Intersection range end // -error getIntersectionRange := func()(bool, [16]byte, [16]byte, error){ if (profileTypeToRetrieve != "Mate"){ // Hosts must host all or none of host/moderator identities, so intersection is always all our requested identities return true, rangeToRetrieveStart, rangeToRetrieveEnd, nil } anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostRangeStart, theirHostRangeEnd) if (err != nil) { return false, [16]byte{}, [16]byte{}, err } if (anyIntersectionFound == false){ return false, [16]byte{}, [16]byte{}, nil } return true, intersectionRangeStart, intersectionRangeEnd, nil } anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := getIntersectionRange() if (err != nil) { return false, [21]byte{}, nil, err } if (anyIntersectionFound == false){ // Host is not hosting any profiles within the range that we are requesting // Skip this host return false, [21]byte{}, nil, nil } exists, hostedProfilesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, profileTypeToRetrieve + "ProfilesQuantity") if (err != nil) { return false, [21]byte{}, nil, err } if (exists == false){ return false, [21]byte{}, nil, errors.New("Database corrupt: Contains host profile missing profilesQuantity") } hostedProfilesQuantityInt64, err := helpers.ConvertStringToInt64(hostedProfilesQuantity) if (err != nil) { return false, [21]byte{}, nil, err } if (hostedProfilesQuantityInt64 == 0){ // Host must have just started their node. Skip them. return false, [21]byte{}, nil, nil } estimatedNumItemsInSubrange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostRangeStart, theirHostRangeEnd, hostedProfilesQuantityInt64, intersectionRangeStart, intersectionRangeEnd) if (err != nil) { return false, [21]byte{}, nil, err } subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInSubrange, int64(maximumProfilesInGetProfilesInfoRequest)) if (err != nil) { return false, [21]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, nil } //Outputs: // -bool: Download successful // -[]serverResponse.ProfileInfoStruct: Retrieved profiles info map list // -error downloadProfileInfoObjectsList := func()(bool, []serverResponse.ProfileInfoStruct, error){ retrievedProfileInfoObjectsList := make([]serverResponse.ProfileInfoStruct, 0) for _, subrangeObject := range subrangesToQueryList{ subrangeStart := subrangeObject.SubrangeStart subrangeEnd := subrangeObject.SubrangeEnd downloadSuccessful, profileInfoObjectsList, err := sendRequests.GetProfilesInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, subrangeStart, subrangeEnd, identityHashesToRetrieveList, criteria, getNewestProfilesOnly, getViewableProfilesOnly) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this subrange. // Could be that host went offline permanently or just this connection failed. // Either way, close connection err := logger.AddLogEntry("Network", "Failed to download profiles info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } retrievedProfileInfoObjectsList = append(retrievedProfileInfoObjectsList, profileInfoObjectsList...) } return true, retrievedProfileInfoObjectsList, nil } downloadSuccessful, retrievedProfileInfoObjectsList, err := downloadProfileInfoObjectsList() if (err != nil){ errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, errB } return false, [21]byte{}, nil, err } if (downloadSuccessful == false){ err = peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, err } return false, [21]byte{}, nil, nil } return true, connectionIdentifier, retrievedProfileInfoObjectsList, nil } downloadSuccessful, connectionIdentifier, retrievedProfilesInfoObjectsList, err := getRetrievedProfileInfoObjectsList() if (err != nil) { return false, err } if (downloadSuccessful == false){ return false, nil } defer peerClient.CloseConnection(connectionIdentifier) if (len(retrievedProfilesInfoObjectsList) == 0){ // Host has no profiles that we need. We are done with this host. return true, nil } profileHashesToDownloadList := make([][28]byte, 0) // Structure: Profile Hash -> Expected Creation Time profileCreationTimesMap := make(map[[28]byte]int64) // Structure: Profile Hash -> Identity Hash profileIdentityHashesMap := make(map[[28]byte][16]byte) hostIsMalicious := false for _, profileInfoObject := range retrievedProfilesInfoObjectsList{ receivedProfileHash := profileInfoObject.ProfileHash receivedProfileAuthor := profileInfoObject.ProfileAuthor receivedProfileCreationTime := profileInfoObject.ProfileCreationTime // Now we check to see if we already have the profile // We check to see if it is known to be outside of requested range/identities list/criteria (host is malicious) // If it is, we still request it in our download, and then set host as malicious afterwards // Otherwise, host would know that we have this profile stored //TODO: Disable this step if only host/moderator mode is enabled storedProfileExists, storedProfileBytes, err := badgerDatabase.GetUserProfile(profileTypeToRetrieve, receivedProfileHash) if (err != nil) { return false, err } checkIfHostIsOfferingInvalidProfile := func()(bool, error){ if (storedProfileExists == false){ return false, nil } ableToRead, storedProfileHash, profileVersion, profileNetworkType, storedProfileAuthor, storedProfileCreationTime, _, storedRawProfileMap, err := readProfiles.ReadProfileAndHash(false, storedProfileBytes) if (err != nil) { return false, err } if (ableToRead == false){ return false, errors.New("Database corrupt: Contains invalid profile.") } if (storedProfileHash != receivedProfileHash){ return false, errors.New("Database corrupt: Contains profile with different hash than entry key.") } if (profileNetworkType != networkType){ return true, nil } if (storedProfileAuthor != receivedProfileAuthor){ return true, nil } if (storedProfileCreationTime != receivedProfileCreationTime){ return true, nil } if (criteria != nil){ criteriaIsValid, fulfillsCriteria, err := mateCriteria.CheckIfMateProfileFulfillsCriteria(false, profileVersion, storedRawProfileMap, criteria) if (err != nil) { return false, err } if (criteriaIsValid == false){ return false, errors.New("DownloadProfilesFromHosts called with invalid criteria.") } if (fulfillsCriteria == false){ return true, nil } } return false, nil } hostIsOfferingInvalidProfile, err := checkIfHostIsOfferingInvalidProfile() if (err != nil) { return false, err } if (hostIsOfferingInvalidProfile == true){ // Host is malicious, we must still download the profile hostIsMalicious = true profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash) profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor profileCreationTimesMap[receivedProfileHash] = receivedProfileCreationTime continue } if (downloadAllMode == true){ // downloadAllMode is enabled, we must download all profiles regardless of if we already have it // This is done to prevent hosts learning our private desires profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash) profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor profileCreationTimesMap[receivedProfileHash] = receivedProfileCreationTime continue } if (storedProfileExists == true){ // We already have profile, skip downloading it if (getViewableProfilesOnly == true){ err = trustedViewableStatus.AddTrustedProfileIsViewableStatus(receivedProfileHash, hostIdentityHash, true) if (err != nil) { return false, err } } continue } // Profile does not exist, we must check if we want to download it // This works differently for Hosting, Moderation, and Browsing shouldDownload, err := checkIfProfileShouldBeDownloaded(receivedProfileHash, receivedProfileAuthor, receivedProfileCreationTime) if (err != nil) { return false, err } if (shouldDownload == true){ profileHashesToDownloadList = append(profileHashesToDownloadList, receivedProfileHash) profileIdentityHashesMap[receivedProfileHash] = receivedProfileAuthor profileCreationTimesMap[receivedProfileHash] = receivedProfileCreationTime continue } } if (len(profileHashesToDownloadList) == 0){ // Nothing to download, done with this host. if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } return true, nil } // Now we retrieve profiles from host maximumProfileHashesInResponse, err := serverResponse.GetMaximumProfilesInResponse_GetProfiles(profileTypeToRetrieve) if (err != nil) { return false, err } profileHashesSublistsList, err := helpers.SplitListIntoSublists(profileHashesToDownloadList, maximumProfileHashesInResponse) if (err != nil) { return false, err } numberOfDownloadedProfiles := 0 for _, profileHashesSublist := range profileHashesSublistsList{ downloadSuccessful, profilesList, err := sendRequests.GetProfilesFromHost(connectionIdentifier, hostIdentityHash, networkType, profileTypeToRetrieve, profileHashesSublist, profileIdentityHashesMap, profileCreationTimesMap, nil) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Request has failed // We will stop downloading from this host break } for _, profileBytes := range profilesList{ //TODO: If downloadAll mode is enabled, we need to check to see if profile should be imported or not (desires pruning mode) profileIsWellFormed, addedProfileHash, err := profileStorage.AddUserProfile(profileBytes) if (err != nil) { return false, err } if (profileIsWellFormed == false){ return false, errors.New("GetProfilesFromHost not verifying profiles are well formed.") } if (getViewableProfilesOnly == true){ err = trustedViewableStatus.AddTrustedProfileIsViewableStatus(addedProfileHash, hostIdentityHash, true) if (err != nil) { return false, err } } } numberOfDownloadedProfiles += len(profilesList) } if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } if (numberOfDownloadedProfiles != 0){ numberOfProfilesDownloadedString := helpers.ConvertIntToString(numberOfDownloadedProfiles) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfProfilesDownloadedString + " " + profileTypeToRetrieve + " profiles from host.") if (err != nil) { return false, err } } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadProfilesFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadMessagesFromHosts( allowClearnet bool, networkType byte, rangeToRetrieveStart [10]byte, rangeToRetrieveEnd [10]byte, inboxesToRetrieveList [][10]byte, getViewableMessagesOnly bool, getDecryptableMessagesOnly bool, checkIfMessageShouldBeDownloaded func([26]byte)(bool, error), numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadMessagesFromHosts called with invalid networkType: " + networkTypeString) } // We cycle through each host // We get all message hashes that they host that are within our requested range/inboxes list // We see if any are knowingly outside of inboxes list, and download anyway if they are (to resist fingerprinting) // We download the messages that we don't already have //TODO: Add better selection of hosts if getViewableOnly is off. Prioritize hosts hosting non-viewable messages //Outputs: // -bool: Successfully downloaded from host (false if host does not have any messages we need) // -error downloadMessagesFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } hostIsHostingMessages, theirInboxRangeStart, theirInboxRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap) if (hostIsHostingMessages == false){ // Host is not hosting any messages. Skip to next host return false, nil } //Outputs: // -bool: Download successful // -[21]byte: Connection identifier // -[][26]byte: Retrieved message hashes list // -[10]byte: Inbox Range start of requested messages // -[10]byte: Inbox range end of requested messages // -[][10]byte: Requested inboxes list (nil if none were requested) // -error getReceivedMessageHashesList := func()(bool, [21]byte, [][26]byte, [10]byte, [10]byte, [][10]byte, error){ // Our request will either be based on inbox range or inbox list // We will split up the request into either ranges or inboxes exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity") if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (exists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile missing MessagesQuantity") } hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } maximumMessageHashesInResponse := serverResponse.MaximumMessageHashesInResponse_GetMessageHashesList - 100 if (len(inboxesToRetrieveList) != 0){ // We will split request into inbox lists, not inbox ranges anyValuesExist, inboxesInMyRangeList, err := byteRange.GetAllInboxesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, inboxesToRetrieveList) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (anyValuesExist == false){ // This should not happen return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Inboxes list to retrieve contains no inboxes within request range.") } anyValuesExist, inboxesInTheirRangeList, err := byteRange.GetAllInboxesInListWithinRange(theirInboxRangeStart, theirInboxRangeEnd, inboxesInMyRangeList) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (anyValuesExist == false){ // This host is not hosting any inboxes we desire. We will skip them. return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } // Now we split request into sublists if needed exists, hostedInboxesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "InboxesQuantity") if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (exists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile missing InboxesQuantity") } hostedInboxesQuantityInt64, err := helpers.ConvertStringToInt64(hostedInboxesQuantity) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid InboxesQuantity: " + hostedInboxesQuantity) } if (hostedInboxesQuantityInt64 == 0){ // Host has no inboxes return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } estimatedNumMessagesPerInbox := hostedMessagesQuantityInt64/hostedInboxesQuantityInt64 maximumInboxesPerRequest := int64(maximumMessageHashesInResponse) / estimatedNumMessagesPerInbox maximumInboxesPerRequestInt, err := helpers.ConvertInt64ToInt(maximumInboxesPerRequest) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } sublistsToQueryList, err := helpers.SplitListIntoSublists(inboxesInTheirRangeList, maximumInboxesPerRequestInt) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds() //Outputs: // -bool: Download successful // -[][26]byte: Message hashes list // -error getMessageHashesListFromHost := func()(bool, [][26]byte, error){ retrievedMessageHashesList := make([][26]byte, 0) for _, inboxesSublist := range sublistsToQueryList{ downloadSuccessful, messageHashesList, err := sendRequests.GetMessageHashesListFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, inboxesSublist, getViewableMessagesOnly, getDecryptableMessagesOnly) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // We are done with this host err := logger.AddLogEntry("Network", "Failed to download message hashes from host.") if (err != nil) { return false, nil, err } return false, nil, nil } retrievedMessageHashesList = append(retrievedMessageHashesList, messageHashesList...) } return true, retrievedMessageHashesList, nil } downloadSuccessful, retrievedMessageHashesList, err := getMessageHashesListFromHost() if (err != nil){ errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errB } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } return true, connectionIdentifier, retrievedMessageHashesList, minimumRange, maximumRange, inboxesInTheirRangeList, nil } // We will retrieve using inbox ranges, not inbox lists anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxRangeStart, theirInboxRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (anyIntersectionFound == false){ // Host is not hosting any messages within the inboxes range that we are requesting // Skip this host return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxRangeStart, theirInboxRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, int64(maximumMessageHashesInResponse)) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } getMessageHashesListFromHost := func()(bool, [][26]byte, error){ retrievedMessageHashesList := make([][26]byte, 0) for _, subrangeObject := range subrangesToQueryList{ subrangeStart := subrangeObject.SubrangeStart subrangeEnd := subrangeObject.SubrangeEnd downloadSuccessful, messageHashesList, err := sendRequests.GetMessageHashesListFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, inboxesToRetrieveList, getViewableMessagesOnly, getDecryptableMessagesOnly) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this subrange. //Could be that host went offline permanently or just this connection failed. // Either way, end connection to host err := logger.AddLogEntry("Network", "Failed to download message hashes from host.") if (err != nil) { return false, nil, err } return false, nil, nil } retrievedMessageHashesList = append(retrievedMessageHashesList, messageHashesList...) } return true, retrievedMessageHashesList, nil } downloadSuccessful, retrievedMessageHashesList, err := getMessageHashesListFromHost() if (err != nil){ errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, errB } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, err } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil, nil } emptyList := make([][10]byte, 0) return true, connectionIdentifier, retrievedMessageHashesList, intersectionRangeStart, intersectionRangeEnd, emptyList, nil } downloadSuccessful, connectionIdentifier, retrievedMessageHashesList, requestedInboxesRangeStart, requestedInboxesRangeEnd, requestedInboxesList, err := getReceivedMessageHashesList() if (err != nil) { return false, err } if (downloadSuccessful == false){ return false, nil } defer peerClient.CloseConnection(connectionIdentifier) // Now we see if we have any of these messages, and if they do not fulfill the request range/inboxes list // If they do not fulfill the request, we know host is malicious // We still download the messages to prevent host from knowing that we have those messages downloaded // We then mark the host as malicious //TODO: Immediately stop connection to host if only Host/Moderator mode is enabled, because fingerprinting is not a risk. messageHashesToDownloadList := make([][26]byte, 0) hostIsMalicious := false for _, receivedMessageHash := range retrievedMessageHashesList{ checkIfHostIsOfferingInvalidMessage := func()(bool, error){ messageMetadataExists, _, messageNetworkType, _, messageInbox, _, err := contentMetadata.GetMessageMetadata(receivedMessageHash) if (err != nil) { return false, err } if (messageMetadataExists == false){ return false, nil } if (messageNetworkType != networkType){ return true, nil } if (len(requestedInboxesList) != 0){ isInInboxesList := slices.Contains(requestedInboxesList, messageInbox) if (isInInboxesList == false){ return true, nil } } else { inboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(requestedInboxesRangeStart, requestedInboxesRangeEnd, messageInbox) if (err != nil) { return false, err } if (inboxIsWithinRange == false){ return true, nil } } return false, nil } hostIsOfferingInvalidMessage, err := checkIfHostIsOfferingInvalidMessage() if (err != nil) { return false, err } if (hostIsOfferingInvalidMessage == true){ // Host is malicious, we must still download the message so they do not learn that we have this message downloaded (or did at one point) hostIsMalicious = true messageHashesToDownloadList = append(messageHashesToDownloadList, receivedMessageHash) continue } storedMessageExists, _, err := badgerDatabase.GetChatMessage(receivedMessageHash) if (err != nil) { return false, err } if (storedMessageExists == true){ // We already have the message, skip downloading it continue } // Message does not exist, we must check if we want to download it // This works differently for Hosting, Moderation, and Viewing // For instance, if the user is downloading messages for viewing, they will not redownload profiles that they have already imported/deleted shouldDownload, err := checkIfMessageShouldBeDownloaded(receivedMessageHash) if (err != nil) { return false, err } if (shouldDownload == true){ messageHashesToDownloadList = append(messageHashesToDownloadList, receivedMessageHash) continue } } if (len(messageHashesToDownloadList) == 0){ // Nothing to download, done with this host. if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } return true, nil } // Now we retrieve messages from host maximumMessagesInResponse := serverResponse.MaximumMessagesInResponse_GetMessages messageHashesSublistsList, err := helpers.SplitListIntoSublists(messageHashesToDownloadList, maximumMessagesInResponse) if (err != nil) { return false, err } numberOfDownloadedMessages := 0 for _, messageHashesSublist := range messageHashesSublistsList{ downloadSuccessful, messagesList, err := sendRequests.GetMessagesFromHost(connectionIdentifier, hostIdentityHash, networkType, messageHashesSublist, requestedInboxesRangeStart, requestedInboxesRangeEnd, requestedInboxesList) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Request has failed // We will stop downloading from this host break } for _, messageBytes := range messagesList{ messageIsWellFormed, err := chatMessageStorage.AddMessage(messageBytes) if (err != nil) { return false, err } if (messageIsWellFormed == false){ return false, errors.New("GetMessagesFromHost not verifying messages are well formed.") } } numberOfDownloadedMessages += len(messagesList) } if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } if (numberOfDownloadedMessages != 0){ numberOfMessagesDownloadedString := helpers.ConvertIntToString(numberOfDownloadedMessages) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfMessagesDownloadedString + " messages from host.") if (err != nil) { return false, err } } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadMessagesFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadIdentityReviewsFromHosts( allowClearnet bool, networkType byte, identityTypeToRetrieve string, rangeToRetrieveStart [16]byte, rangeToRetrieveEnd [16]byte, reviewedIdentityHashesToRetrieveList [][16]byte, reviewersToRetrieveList [][16]byte, checkIfReviewShouldBeDownloaded func([29]byte, []byte)(bool, error), numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadIdentityReviewsFromHosts called with invalid networkType: " + networkTypeString) } //TODO: Verify inputs //Outputs: // -bool: Download successful // -error downloadReviewsFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } // Outputs: // -bool: Successfully downloaded from host // -[21]byte: Connection identifier // -map[[29]byte][]byte: Map of review info: ReviewHash -> ReviewedHash // -[16]byte: Expected Identity Start Range of retrieved reviews // -[16]byte: Expected Identity End range of retrieved reviews // -[][16]byte: Expected retrieved identity hashes list // -error getRetrievedReviewsInfoMap := func()(bool, [21]byte, map[[29]byte][]byte, [16]byte, [16]byte, [][16]byte, error){ hostIsHostingIdentityType, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, identityTypeToRetrieve) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostIsHostingIdentityType == false){ // Host is not hosting any profiles/reviews of this identityType. Skip to next host return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } exists, identityTypeIdentitiesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, identityTypeToRetrieve + "IdentitiesQuantity") if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (exists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "IdentitiesQuantity") } identityTypeIdentitiesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeIdentitiesQuantity) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity) } getMaximumIdentitiesPerRequest := func()(int64, error){ maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetIdentityReviewsInfo if (len(reviewersToRetrieveList) != 0){ // We assume each reviewer has reviewed each identity and all it's profiles profilesQuantityAttributeName := identityTypeToRetrieve + "ProfilesQuantity" exists, identityTypeProfilesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, profilesQuantityAttributeName) if (err != nil) { return 0, err } if (exists == false){ return 0, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ProfilesQuantity") } identityTypeProfilesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeProfilesQuantity) if (err != nil) { return 0, err } estimatedNumberOfProfilesPerIdentity := identityTypeProfilesQuantityInt64 / identityTypeIdentitiesQuantityInt64 numberOfReviewersToRetrieve := int64(len(reviewersToRetrieveList)) estimatedReviewsPerIdentity := numberOfReviewersToRetrieve * (estimatedNumberOfProfilesPerIdentity + 1) maximumIdentitiesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerIdentity return maximumIdentitiesPerRequest, nil } reviewsQuantityAttributeName := identityTypeToRetrieve + "ReviewsQuantity" exists, identityTypeReviewsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, reviewsQuantityAttributeName) if (err != nil) { return 0, err } if (exists == false){ return 0, errors.New("Database corrupt: Contains host profile missing " + identityTypeToRetrieve + "ReviewsQuantity") } identityTypeReviewsQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeReviewsQuantity) if (err != nil) { return 0, err } estimatedReviewsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReviewsQuantityInt64 maximumIdentitiesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerIdentity return maximumIdentitiesPerRequest, nil } maximumIdentitiesPerRequest, err := getMaximumIdentitiesPerRequest() if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } // Our request will either be based on identity hash range or identity hashes list // We will split up the request into either ranges or identity hashes if (len(reviewedIdentityHashesToRetrieveList) != 0){ // We will split the request into identity hash lists hostingAny, identitiesInHostRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, reviewedIdentityHashesToRetrieveList) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostingAny == false){ // Host is not hosting any of our desired identities. Skip host. return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } // Below step is just being overly cautious. // The GetIdentityReviewsfromHosts function should request either a list of identities or a range. Not both. hostingAny, identitiesInRequestRange, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identitiesInHostRangeList) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostingAny == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Requested review identity hashes list outside of request range") } sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInRequestRange, int(maximumIdentitiesPerRequest)) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds() getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][]byte, error){ retrievedReviewsInfoMap := make(map[[29]byte][]byte) for _, identityHashesSublist := range sublistsToQueryList{ downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetIdentityReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist, reviewersToRetrieveList) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, stop download from host. err := logger.AddLogEntry("Network", "Failed to download identity reviews info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{ retrievedReviewsInfoMap[reviewHash] = reviewedHash } } return true, retrievedReviewsInfoMap, nil } downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } return true, connectionIdentifier, retrievedReviewsInfoMap, minimumRange, maximumRange, identitiesInRequestRange, nil } // We will retrieve based on identity range, not identities list anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (anyIntersectionFound == false){ // Host is not hosting any reviews within the identites range that we are requesting // Skip this host return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } estimatedNumIdentitiesInIntersectionRange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, identityTypeIdentitiesQuantityInt64, intersectionRangeStart, intersectionRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumIdentitiesInIntersectionRange, maximumIdentitiesPerRequest) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][]byte, error){ retrievedReviewsInfoMap := make(map[[29]byte][]byte) for _, subrangeObject := range subrangesToQueryList{ subrangeStart := subrangeObject.SubrangeStart subrangeEnd := subrangeObject.SubrangeEnd emptyList := make([][16]byte, 0) downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetIdentityReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, subrangeStart, subrangeEnd, emptyList, reviewersToRetrieveList) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, stop download. err := logger.AddLogEntry("Network", "Failed to download identity reviews info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{ retrievedReviewsInfoMap[reviewHash] = reviewedHash } } return true, retrievedReviewsInfoMap, nil } downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } return true, connectionIdentifier, retrievedReviewsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil, nil } hostDownloadSuccessful, connectionIdentifier, retrievedReviewsInfoMap, expectedRangeStart, expectedRangeEnd, expectedReviewedIdentityHashesList, err := getRetrievedReviewsInfoMap() if (err != nil) { return false, err } if (hostDownloadSuccessful == false){ // We did not download any reviews from this host. Skip them. return false, nil } defer peerClient.CloseConnection(connectionIdentifier) // Now we see if we have any of these reviews, and if they do not fulfill the requests parameters // If they do not fulfill the request, we know host is malicious // This requires checking against content stored in our database // A host could use this to learn information about what content we are storing // We still download the reviews to prevent host from knowing that we have the reviewed content downloaded // Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response // Only after the full request is finished do we mark the host as malicious // TODO: We shouldn't have to wait until the end of the request if we are only in Host/Moderator mode // This is because we don't need to worry about leaking what content we have stored reviewHashesToDownloadList := make([][29]byte, 0) hostIsMalicious := false for receivedReviewHash, receivedReviewedHash := range retrievedReviewsInfoMap{ storedReviewExists, storedReviewBytes, err := badgerDatabase.GetReview(receivedReviewHash) if (err != nil) { return false, err } checkIfHostIsOfferingInvalidReview := func()(bool, error){ if (storedReviewExists == true){ ableToRead, storedReviewHash, _, reviewNetworkType, storedReviewReviewerIdentity, _, storedReviewType, storedReviewReviewedHash, _, _, err := readReviews.ReadReviewAndHash(false, storedReviewBytes) if (err != nil) { return false, err } if (ableToRead == false){ return false, errors.New("Database corrupt: Contains invalid review.") } if (storedReviewHash != receivedReviewHash){ return false, errors.New("Database corrupt: Contains review with invalid entry key hash") } if (reviewNetworkType != networkType){ return true, nil } if (len(reviewersToRetrieveList) != 0){ isInList := slices.Contains(reviewersToRetrieveList, storedReviewReviewerIdentity) if (isInList == false){ return true, nil } } areEqual := bytes.Equal(storedReviewReviewedHash, receivedReviewedHash) if (areEqual == false){ return true, nil } if (storedReviewType != "Identity" && storedReviewType != "Profile" && storedReviewType != "Attribute"){ return true, nil } } // Now we check to see if we are aware of the reviewed content's metadata, and if it matches our request parameters receivedReviewedType, err := helpers.GetReviewedTypeFromReviewedHash(receivedReviewedHash) if (err != nil) { return false, err } if (receivedReviewedType == "Profile"){ // We must verify that the profile's identity hash is expected // We don't have to do this for identities, because we already checked that in the GetIdentityReviewsInfo function if (len(receivedReviewedHash) != 28){ receivedReviewedHashHex := encoding.EncodeBytesToHexString(receivedReviewedHash) return false, errors.New("GetReviewedTypeFromReviewedHash returning Profile for different length reviewedHash: " + receivedReviewedHashHex) } receivedReviewedProfileHash := [28]byte(receivedReviewedHash) metadataIsKnown, _, profileNetworkType, reviewedProfileAuthor, _, profileIsDisabled, _, _, err := contentMetadata.GetProfileMetadata(receivedReviewedProfileHash) if (err != nil) { return false, err } if (metadataIsKnown == false){ return false, nil } if (profileIsDisabled == true){ // Disabled profiles cannot be reviewed. Host must be malicious. return true, nil } if (profileNetworkType != networkType){ // Review is reviewing profile from different networkType // Host should have checked this already. Host must be malicious. return true, nil } if (expectedReviewedIdentityHashesList != nil){ identityIsInList := slices.Contains(expectedReviewedIdentityHashesList, reviewedProfileAuthor) if (identityIsInList == false){ return true, nil } } else { identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, reviewedProfileAuthor) if (err != nil) { return false, err } if (identityIsWithinRange == false){ return true, nil } } } if (receivedReviewedType == "Attribute"){ if (len(receivedReviewedHash) != 27){ receivedReviewedHashHex := encoding.EncodeBytesToHexString(receivedReviewedHash) return false, errors.New("GetReviewedTypeFromReviewedHash returning Attribute for different length reviewedHash: " + receivedReviewedHashHex) } receivedReviewedAttributeHash := [27]byte(receivedReviewedHash) metadataExists, _, attributeAuthorIdentityHash, attributeNetworkType, _, err := profileStorage.GetProfileAttributeMetadata(receivedReviewedAttributeHash) if (err != nil){ return false, err } if (metadataExists == true){ if (expectedReviewedIdentityHashesList != nil){ identityIsInList := slices.Contains(expectedReviewedIdentityHashesList, attributeAuthorIdentityHash) if (identityIsInList == false){ return true, nil } } else { identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, attributeAuthorIdentityHash) if (err != nil) { return false, err } if (identityIsWithinRange == false){ return true, nil } } if (attributeNetworkType != networkType){ // Review is reviewing attribute from different networkType // Host should have checked this already. Host must be malicious. return true, nil } return false, nil } } // Review is not invalid, as far as we know return false, nil } hostIsOfferingInvalidReview, err := checkIfHostIsOfferingInvalidReview() if (err != nil) { return false, err } if (hostIsOfferingInvalidReview == true){ // Host is malicious, we must still download the review hostIsMalicious = true reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash) continue } if (storedReviewExists == true){ // We already have the review, skip downloading it continue } // Review does not exist, we must check if we want to download it shouldDownload, err := checkIfReviewShouldBeDownloaded(receivedReviewHash, receivedReviewedHash) if (err != nil) { return false, err } if (shouldDownload == true){ reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash) continue } } if (len(reviewHashesToDownloadList) == 0){ // Nothing to download, we are done with this host. if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } return true, nil } // Now we download reviews maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetReviews reviewHashesSublistsList, err := helpers.SplitListIntoSublists(reviewHashesToDownloadList, maximumReviewsInResponse) if (err != nil) { return false, err } numberOfDownloadedReviews := 0 for _, reviewHashesSublist := range reviewHashesSublistsList{ downloadSuccessful, reviewsList, err := sendRequests.GetReviewsFromHost(connectionIdentifier, hostIdentityHash, networkType, reviewHashesSublist, retrievedReviewsInfoMap, reviewersToRetrieveList) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Request has failed // We will stop downloading from this host break } if (hostIsMalicious == true){ // We don't add reviews from malicious hosts continue } for _, reviewBytes := range reviewsList{ reviewIsWellFormed, err := reviewStorage.AddReview(reviewBytes) if (err != nil) { return false, err } if (reviewIsWellFormed == false){ return false, errors.New("GetReviewsFromHost not verifying reviews are well formed.") } } numberOfDownloadedReviews += len(reviewsList) } if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } if (numberOfDownloadedReviews != 0){ numberOfReviewsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReviews) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReviewsDownloadedString + " identity reviews from host.") if (err != nil) { return false, err } } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadReviewsFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadMessageReviewsFromHosts( allowClearnet bool, networkType byte, rangeToRetrieveStart [10]byte, rangeToRetrieveEnd [10]byte, reviewedMessageHashesToRetrieveList [][26]byte, reviewersToRetrieveList [][16]byte, reviewedMessageInboxesMap map[[26]byte][10]byte, // Message hash -> Message inbox checkIfReviewShouldBeDownloaded func([29]byte, [26]byte)(bool, error), numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadMessageReviewsFromHosts called with invalid networkType: " + networkTypeString) } //TODO: Verify inputs //Outputs: // -bool: Download successful // -error downloadReviewsFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } // Outputs: // -bool: Successfully downloaded from host // -[21]byte: Connection identifier // -map[[29]byte][26]byte: Map of review info: ReviewHash -> ReviewedHash // -[10]byte: Expected Inbox Start Range of retrieved reviews // -[10]byte: Expected Inbox End range of retrieved reviews // -error getRetrievedReviewsInfoMap := func()(bool, [21]byte, map[[29]byte][26]byte, [10]byte, [10]byte, error){ hostIsHostingMessages, theirInboxRangeStart, theirInboxRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (hostIsHostingMessages == false){ // Host is not hosting messages. Nothing to download. return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity") if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (exists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessagesQuantity") } hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity) } exists, hostedMessageReviewsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessageReviewsQuantity") if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (exists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessageReviewsQuantity") } hostedMessageReviewsQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessageReviewsQuantity) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessageReviewsQuantity: " + hostedMessageReviewsQuantity) } getMaximumMessageHashesPerRequest := func()int{ maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetMessageReviewsInfo if (len(reviewersToRetrieveList) != 0){ // We assume each reviewer has 1 review per message numberOfReviewersToRetrieve := len(reviewersToRetrieveList) maximumMessageHashesPerRequest := maximumReviewsInResponse / numberOfReviewersToRetrieve return maximumMessageHashesPerRequest } // We get the estimated number of reviews per message estimatedReviewsPerMessage := hostedMessageReviewsQuantityInt64 / hostedMessagesQuantityInt64 maximumMessageHashesPerRequest := int64(maximumReviewsInResponse) / estimatedReviewsPerMessage return int(maximumMessageHashesPerRequest) } maximumMessageHashesPerRequest := getMaximumMessageHashesPerRequest() // We will retrieve reviews from either the reviewed hashes list, or request range if (len(reviewedMessageHashesToRetrieveList) != 0){ // We will retrieve reviewed hashes list messageHashesInTheirRangeList := make([][26]byte, 0) for _, messageHash := range reviewedMessageHashesToRetrieveList{ messageInbox, exists := reviewedMessageInboxesMap[messageHash] if (exists == false) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("reviewedMessageInboxesMap missing message hash.") } inboxIsWithinTheirRange, err := byteRange.CheckIfInboxIsWithinRange(theirInboxRangeStart, theirInboxRangeEnd, messageInbox) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (inboxIsWithinTheirRange == true){ messageHashesInTheirRangeList = append(messageHashesInTheirRangeList, messageHash) } } sublistsToQueryList, err := helpers.SplitListIntoSublists(messageHashesInTheirRangeList, maximumMessageHashesPerRequest) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds() hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } //Outputs: // -bool: Download successful // -map[[29]byte][26]byte: Reviews info map (Review hash -> Reviewed message hash) // -error getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][26]byte, error){ retrievedReviewsInfoMap := make(map[[29]byte][26]byte) for _, messageHashesSublist := range sublistsToQueryList{ downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetMessageReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, messageHashesSublist, reviewersToRetrieveList) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, stop retrieving from this host err := logger.AddLogEntry("Network", "Failed to download message reviews info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{ retrievedReviewsInfoMap[reviewHash] = reviewedHash } } return true, retrievedReviewsInfoMap, nil } downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap() if (err != nil){ errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } return true, connectionIdentifier, retrievedReviewsInfoMap, minimumRange, maximumRange, nil } // We will retrieve reviews based on message inbox range anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxRangeStart, theirInboxRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (anyIntersectionFound == false){ // Host is not hosting any reviews within the inboxes range that we are requesting // Skip this host return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxRangeStart, theirInboxRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, int64(maximumMessageHashesPerRequest)) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } //Outputs: // -bool: Download successful // -map[[29]byte][26]byte: Downloaded reviews info map (Review Hash -> Reviewed Message Hash) // -error getDownloadedReviewsInfoMap := func()(bool, map[[29]byte][26]byte, error){ retrievedReviewsInfoMap := make(map[[29]byte][26]byte) for _, subrangeObject := range subrangesToQueryList{ subrangeStart := subrangeObject.SubrangeStart subrangeEnd := subrangeObject.SubrangeEnd emptyList := make([][26]byte, 0) downloadSuccessful, currentRetrievedReviewsInfoMap, err := sendRequests.GetMessageReviewsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, emptyList, reviewersToRetrieveList) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, we are done with this host err := logger.AddLogEntry("Network", "Failed to download message reviews info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reviewHash, reviewedHash := range currentRetrievedReviewsInfoMap{ retrievedReviewsInfoMap[reviewHash] = reviewedHash } } return true, retrievedReviewsInfoMap, nil } downloadSuccessful, retrievedReviewsInfoMap, err := getDownloadedReviewsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } return true, connectionIdentifier, retrievedReviewsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil } hostDownloadSuccessful, connectionIdentifier, retrievedReviewsInfoMap, expectedRangeStart, expectedRangeEnd, err := getRetrievedReviewsInfoMap() if (err != nil) { return false, err } if (hostDownloadSuccessful == false){ // We did not download any reviews from this host. Skip them. return false, nil } defer peerClient.CloseConnection(connectionIdentifier) // Now we see if we have any of these reviews, and if they do not fulfill the requests parameters // If they do not fulfill the request, we know host is malicious // This requires checking against content stored in our database // A host could use this to learn information about what messages we are storing // We still download the reviews to prevent host from knowing that we have the reviewed messages downloaded // Otherwise they could see that we stopped our request mid-way and become aware of the fact that we detected their malicious response // Only after the full request is finished do we mark the host as malicious // TODO: We shouldn't have to wait until the end of the request if we are only in Host mode // This is because if we we don't need to worry about leaking what messages we have stored reviewHashesToDownloadList := make([][29]byte, 0) hostIsMalicious := false for receivedReviewHash, receivedReviewedHash := range retrievedReviewsInfoMap{ storedReviewExists, storedReviewBytes, err := badgerDatabase.GetReview(receivedReviewHash) if (err != nil) { return false, err } checkIfHostIsOfferingInvalidReview := func()(bool, error){ if (storedReviewExists == true){ ableToRead, storedReviewHash, _, reviewNetworkType, storedReviewReviewerIdentity, _, storedReviewType, storedReviewReviewedHash, _, _, err := readReviews.ReadReviewAndHash(false, storedReviewBytes) if (err != nil) { return false, err } if (ableToRead == false){ return false, errors.New("Database corrupt: Contains invalid review.") } if (storedReviewHash != receivedReviewHash){ return false, errors.New("Database corrupt: Contains review with invalid entry key hash") } if (reviewNetworkType != networkType){ return true, nil } if (len(reviewersToRetrieveList) != 0){ isInList := slices.Contains(reviewersToRetrieveList, storedReviewReviewerIdentity) if (isInList == false){ return true, nil } } areEqual := bytes.Equal(storedReviewReviewedHash, receivedReviewedHash[:]) if (areEqual == false){ return true, nil } if (storedReviewType != "Message"){ return true, nil } } // Now we check to see if we are aware of the reviewed message's metadata, and if it matches our request parameters metadataIsKnown, _, messageNetworkType, _, reviewedMessageInbox, _, err := contentMetadata.GetMessageMetadata(receivedReviewedHash) if (err != nil) { return false, err } if (metadataIsKnown == false){ return false, nil } if (messageNetworkType != networkType){ // The review is reviewing a message from a different networkType // The host should have checked for this before serving us the review // Thus, the host must be malicious return true, nil } messageInboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(expectedRangeStart, expectedRangeEnd, reviewedMessageInbox) if (err != nil) { return false, err } if (messageInboxIsWithinRange == false){ return false, nil } return false, nil } hostIsOfferingInvalidReview, err := checkIfHostIsOfferingInvalidReview() if (err != nil) { return false, err } if (hostIsOfferingInvalidReview == true){ // Host is malicious, we must still download the review hostIsMalicious = true reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash) continue } if (storedReviewExists == true){ // We already have the review, skip downloading it continue } // Review does not exist, we must check if we want to download it shouldDownload, err := checkIfReviewShouldBeDownloaded(receivedReviewHash, receivedReviewedHash) if (err != nil) { return false, err } if (shouldDownload == true){ reviewHashesToDownloadList = append(reviewHashesToDownloadList, receivedReviewHash) continue } } if (len(reviewHashesToDownloadList) == 0){ // Nothing to download, done with this host. if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } return true, nil } // Now we download reviews // We convert the [26]byte values to []byte values to pass to GetReviewsFromHost reviewsInfoMapForRequest := make(map[[29]byte][]byte) for reviewHash, reviewedMessageHash := range retrievedReviewsInfoMap{ reviewsInfoMapForRequest[reviewHash] = reviewedMessageHash[:] } maximumReviewsInResponse := serverResponse.MaximumReviewsInResponse_GetReviews reviewHashesSublistsList, err := helpers.SplitListIntoSublists(reviewHashesToDownloadList, maximumReviewsInResponse) if (err != nil) { return false, err } numberOfDownloadedReviews := 0 for _, reviewHashesSublist := range reviewHashesSublistsList{ downloadSuccessful, reviewsList, err := sendRequests.GetReviewsFromHost(connectionIdentifier, hostIdentityHash, networkType, reviewHashesSublist, reviewsInfoMapForRequest, reviewersToRetrieveList) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Request has failed // We will stop downloading from this host break } if (hostIsMalicious == true){ // We don't add reviews from malicious hosts continue } for _, reviewBytes := range reviewsList{ reviewIsWellFormed, err := reviewStorage.AddReview(reviewBytes) if (err != nil) { return false, err } if (reviewIsWellFormed == false){ return false, errors.New("GetReviewsFromHost not verifying reviews are well formed.") } } numberOfDownloadedReviews += len(reviewsList) } if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } if (numberOfDownloadedReviews != 0){ numberOfReviewsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReviews) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReviewsDownloadedString + " reviews from host.") if (err != nil) { return false, err } } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadReviewsFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadIdentityReportsFromHosts( allowClearnet bool, networkType byte, identityTypeToRetrieve string, rangeToRetrieveStart [16]byte, rangeToRetrieveEnd [16]byte, reportedIdentityHashesToRetrieveList [][16]byte, checkIfReportShouldBeDownloaded func([30]byte, []byte)(bool, error), numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadIdentityReportsFromHosts called with invalid networkType: " + networkTypeString) } //TODO: Verify inputs //Outputs: // -bool: Download successful // -error downloadReportsFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } // Outputs: // -bool: Successfully downloaded from host // -[21]byte: Connection identifier // -map[[30]byte][]byte: Map of report info: ReportHash -> ReportedHash // -[16]byte: Expected Identity Start Range of retrieved reports // -[16]byte: Expected Identity End range of retrieved reports // -[][16]byte: Expected retrieved identity hashes list // -error getRetrievedReportsInfoMap := func()(bool, [21]byte, map[[30]byte][]byte, [16]byte, [16]byte, [][16]byte, error){ hostIsHostingIdentityType, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, identityTypeToRetrieve) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostIsHostingIdentityType == false){ // Host is not hosting any profiles/reports of this identityType. Skip to next host return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } identitiesQuantityAttributeName := identityTypeToRetrieve + "IdentitiesQuantity" exists, identityTypeIdentitiesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, identitiesQuantityAttributeName) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (exists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + identitiesQuantityAttributeName) } identityTypeIdentitiesQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeIdentitiesQuantity) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains Host profile with invalid " + identityTypeToRetrieve + "IdentitiesQuantity: " + identityTypeIdentitiesQuantity) } reportsQuantityAttributeName := identityTypeToRetrieve + "ReportsQuantity" exists, identityTypeReportsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, reportsQuantityAttributeName) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (exists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile missing " + reportsQuantityAttributeName) } identityTypeReportsQuantityInt64, err := helpers.ConvertStringToInt64(identityTypeReportsQuantity) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Database corrupt: Contains host profile with invalid " + reportsQuantityAttributeName + ": " + identityTypeReportsQuantity) } //TODO: Deal with divide by 0 case here and on all other functions estimatedReportsPerIdentity := identityTypeIdentitiesQuantityInt64 / identityTypeReportsQuantityInt64 maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetIdentityReportsInfo maximumIdentitiesPerRequest := int64(maximumReportsInResponse) / estimatedReportsPerIdentity // Our request will either be based on identity hash range or identity hashes list // We will split up the request into either ranges or identity hash lists if (len(reportedIdentityHashesToRetrieveList) != 0){ // We will split the request into identity hash lists hostingAny, identitiesInHostRangeList, err := byteRange.GetAllIdentityHashesInListWithinRange(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, reportedIdentityHashesToRetrieveList) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostingAny == false){ // Host is not hosting any of our desired identities. Skip host. return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } // Below step is just being overly cautious. // The GetMessageReportsfromHosts function should request either a list of identities or a range. Not both. hostingAny, identitiesInRequestRange, err := byteRange.GetAllIdentityHashesInListWithinRange(rangeToRetrieveStart, rangeToRetrieveEnd, identitiesInHostRangeList) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostingAny == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errors.New("Requested report identity hashes list outside of request range") } sublistsToQueryList, err := helpers.SplitListIntoSublists(identitiesInRequestRange, int(maximumIdentitiesPerRequest)) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } minimumRange, maximumRange := byteRange.GetMinimumMaximumIdentityHashBounds() //Outputs: // -bool: Download successful // -map[[30]byte][]byte: Downloaded reports info map (Report Hash -> Reported Hash) // -error getDownloadedReportsInfoMap := func()(bool, map[[30]byte][]byte, error){ retrievedReportsInfoMap := make(map[[30]byte][]byte) for _, identityHashesSublist := range sublistsToQueryList{ downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetIdentityReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, minimumRange, maximumRange, identityHashesSublist) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, stop downloading from this host err := logger.AddLogEntry("Network", "Failed to download identity reports info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reportHash, reportedHash := range currentRetrievedReportsInfoMap{ retrievedReportsInfoMap[reportHash] = reportedHash } } return true, retrievedReportsInfoMap, nil } downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } return true, connectionIdentifier, retrievedReportsInfoMap, minimumRange, maximumRange, identitiesInRequestRange, nil } // We will retrieve based on identity range, not identities list anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetIdentityIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (anyIntersectionFound == false){ // Host is not hosting any reports within the identites range that we are requesting // Skip this host return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } estimatedNumIdentitiesInIntersectionRange, err := byteRange.GetEstimatedIdentitySubrangeQuantity(theirHostedIdentitiesRangeStart, theirHostedIdentitiesRangeEnd, identityTypeIdentitiesQuantityInt64, intersectionRangeStart, intersectionRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } subrangesToQueryList, err := byteRange.SplitIdentityRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumIdentitiesInIntersectionRange, maximumIdentitiesPerRequest) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } getDownloadedReportsInfoMap := func()(bool, map[[30]byte][]byte, error){ retrievedReportsInfoMap := make(map[[30]byte][]byte) for _, subrangeObject := range subrangesToQueryList{ subrangeStart := subrangeObject.SubrangeStart subrangeEnd := subrangeObject.SubrangeEnd emptyList := make([][16]byte, 0) downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetIdentityReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, identityTypeToRetrieve, subrangeStart, subrangeEnd, emptyList) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, close connection to host. err := logger.AddLogEntry("Network", "Failed to download identity reports info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reportHash, reportedHash := range currentRetrievedReportsInfoMap{ retrievedReportsInfoMap[reportHash] = reportedHash } } return true, retrievedReportsInfoMap, nil } downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, errB } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, err } return false, [21]byte{}, nil, [16]byte{}, [16]byte{}, nil, nil } return true, connectionIdentifier, retrievedReportsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil, nil } hostDownloadSuccessful, connectionIdentifier, retrievedReportsInfoMap, expectedRangeStart, expectedRangeEnd, expectedReportedIdentityHashesList, err := getRetrievedReportsInfoMap() if (err != nil) { return false, err } if (hostDownloadSuccessful == false){ // We did not download any reports from this host. Skip them. return false, nil } defer peerClient.CloseConnection(connectionIdentifier) // Now we see if we have any of these reports, and if they do not fulfill the requests parameters // If they do not fulfill the request, we know host is malicious // We still download the reports to prevent host from knowing that we have those reports downloaded // We then mark the host as malicious reportHashesToDownloadList := make([][30]byte, 0) hostIsMalicious := false for receivedReportHash, receivedReportedHash := range retrievedReportsInfoMap{ //TODO: Ability to disable this step if only 1 of Host/Moderator mode is enabled storedReportExists, storedReportBytes, err := badgerDatabase.GetReport(receivedReportHash) if (err != nil) { return false, err } checkIfHostIsOfferingInvalidReport := func()(bool, error){ if (storedReportExists == true){ ableToRead, storedReportHash, _, reportNetworkType, _, storedReportType, storedReportReportedHash, _, err := readReports.ReadReportAndHash(false, storedReportBytes) if (err != nil) { return false, err } if (ableToRead == false){ return false, errors.New("Database corrupt: Contains invalid report") } if (storedReportHash != receivedReportHash){ return false, errors.New("Database corrupt: Contains report with invalid entry key hash") } if (reportNetworkType != networkType){ return true, nil } areEqual := bytes.Equal(storedReportReportedHash, receivedReportedHash) if (areEqual == false){ return true, nil } if (storedReportType != "Identity" && storedReportType != "Profile" && storedReportType != "Attribute"){ return true, nil } } // Now we see if we have the reviewed profile metadata reportedType, err := helpers.GetReportedTypeFromReportedHash(receivedReportedHash) if (err != nil){ return false, err } if (reportedType == "Profile"){ if (len(receivedReportedHash) != 28){ reportedHashHex := encoding.EncodeBytesToHexString(receivedReportedHash) return false, errors.New("GetReportedTypeFromReportedHash returning Profile for different length reportedHash: " + reportedHashHex) } receivedReportedProfileHash := [28]byte(receivedReportedHash) metadataExists, _, profileNetworkType, profileAuthor, _, profileIsDisabled, _, _, err := contentMetadata.GetProfileMetadata(receivedReportedProfileHash) if (err != nil) { return false, err } if (metadataExists == false){ return false, nil } if (profileNetworkType != networkType){ // Reported profile belongs to a different networkType than the report // The host should have checked for this before serving this report // Host must be malicious return true, nil } if (profileIsDisabled == true){ // Disabled profiles cannot be reported // Host must be malicious return true, nil } if (expectedReportedIdentityHashesList != nil){ identityIsInList := slices.Contains(expectedReportedIdentityHashesList, profileAuthor) if (identityIsInList == false){ return true, nil } } else { identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, profileAuthor) if (err != nil) { return false, err } if (identityIsWithinRange == false){ return true, nil } } } else if (reportedType == "Attribute"){ if (len(receivedReportedHash) != 27){ reportedHashHex := encoding.EncodeBytesToHexString(receivedReportedHash) return false, errors.New("GetReportedTypeFromReportedHash returning Attribute for different length reportedHash: " + reportedHashHex) } receivedReportedAttributeHash := [27]byte(receivedReportedHash) metadataExists, _, attributeAuthorIdentityHash, attributeNetworkType, _, err := profileStorage.GetProfileAttributeMetadata(receivedReportedAttributeHash) if (err != nil){ return false, err } if (metadataExists == true){ if (expectedReportedIdentityHashesList != nil){ identityIsInList := slices.Contains(expectedReportedIdentityHashesList, attributeAuthorIdentityHash) if (identityIsInList == false){ return true, nil } } else { identityIsWithinRange, err := byteRange.CheckIfIdentityHashIsWithinRange(expectedRangeStart, expectedRangeEnd, attributeAuthorIdentityHash) if (err != nil) { return false, err } if (identityIsWithinRange == false){ return true, nil } } if (attributeNetworkType != networkType){ // Report is reporting attribute from different networkType // Host should have checked this already. Host must be malicious. return true, nil } return false, nil } } // Host is not offering invalid report (as far as we know) return false, nil } hostIsOfferingInvalidReport, err := checkIfHostIsOfferingInvalidReport() if (err != nil) { return false, err } if (hostIsOfferingInvalidReport == true){ // Host is malicious, we must still download the report hostIsMalicious = true reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash) continue } if (storedReportExists == true){ // We already have the report, skip downloading it continue } // Report does not exist, we must check if we want to download it // This works differently for Hosting and Moderation shouldDownload, err := checkIfReportShouldBeDownloaded(receivedReportHash, receivedReportedHash) if (err != nil) { return false, err } if (shouldDownload == true){ reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash) continue } } if (len(reportHashesToDownloadList) == 0){ // Nothing to download, done with this host. if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } return true, nil } // Now we download reports maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetReports reportHashesSublistsList, err := helpers.SplitListIntoSublists(reportHashesToDownloadList, maximumReportsInResponse) if (err != nil) { return false, err } numberOfDownloadedReports := 0 for _, reportHashesSublist := range reportHashesSublistsList{ downloadSuccessful, reportsList, err := sendRequests.GetReportsFromHost(connectionIdentifier, hostIdentityHash, networkType, reportHashesSublist, retrievedReportsInfoMap) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Request has failed // We will stop downloading from this host break } for _, reportBytes := range reportsList{ reportIsWellFormed, err := reportStorage.AddReport(reportBytes) if (err != nil) { return false, err } if (reportIsWellFormed == false){ return false, errors.New("GetReportsFromHost not verifying reports are well formed.") } } numberOfDownloadedReports += len(reportsList) } if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } if (numberOfDownloadedReports != 0){ numberOfReportsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReports) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReportsDownloadedString + " identity reports from host.") if (err != nil) { return false, err } } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadReportsFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadMessageReportsFromHosts( allowClearnet bool, networkType byte, rangeToRetrieveStart [10]byte, rangeToRetrieveEnd [10]byte, reportedMessageHashesToRetrieveList [][26]byte, reportedMessagesInboxMap map[[26]byte][10]byte, // Message hash -> Message inbox checkIfReportShouldBeDownloaded func([30]byte, [26]byte)(bool, error), numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadMessageReportsFromHosts called with invalid networkType: " + networkTypeString) } //TODO: Verify inputs //Outputs: // -bool: Download successful // -error downloadReportsFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } // Outputs: // -bool: Successfully downloaded from host // -[21]byte: Connection identifier // -map[[30]byte][26]byte: Map of report info: ReportHash-> Reported message Hash // -[10]byte: Expected Inbox Start Range of retrieved reports // -[10]byte: Expected Inbox End range of retrieved reports // -error getRetrievedReportsInfoMap := func()(bool, [21]byte, map[[30]byte][26]byte, [10]byte, [10]byte, error){ hostIsHostingMessages, theirInboxHostRangeStart, theirInboxHostRangeEnd, err := hostRanges.GetHostedMessageInboxesRangeFromHostRawProfileMap(hostRawProfileMap) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (hostIsHostingMessages == false){ // Host is not hosting messages. Nothing to download. return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } exists, hostedMessagesQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessagesQuantity") if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (exists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessagesQuantity") } hostedMessagesQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessagesQuantity) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile with invalid MessagesQuantity: " + hostedMessagesQuantity) } exists, hostedMessageReportsQuantity, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "MessageReportsQuantity") if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (exists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Database corrupt: Contains host profile missing MessageReportsQuantity") } hostedMessageReportsQuantityInt64, err := helpers.ConvertStringToInt64(hostedMessageReportsQuantity) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("Databse corrupt: Contains host profile with invalid MessageReportsQuantity: " + hostedMessageReportsQuantity) } if (hostedMessageReportsQuantityInt64 == 0){ // Host is not hosting any message reports. Nothing to download. return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetMessageReportsInfo estimatedReportsPerMessage := hostedMessageReportsQuantityInt64 / hostedMessagesQuantityInt64 maximumMessageHashesPerRequest := int64(maximumReportsInResponse) / estimatedReportsPerMessage // We will retrieve reports from either the reported hashes list, or request range if (len(reportedMessageHashesToRetrieveList) != 0){ // We will retrieve reported message hashes list messageHashesInTheirRangeList := make([][26]byte, 0) for _, messageHash := range reportedMessageHashesToRetrieveList{ messageInbox, exists := reportedMessagesInboxMap[messageHash] if (exists == false) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errors.New("ReportedMessagesInboxMap missing message hash.") } inboxIsWithinTheirRange, err := byteRange.CheckIfInboxIsWithinRange(theirInboxHostRangeStart, theirInboxHostRangeEnd, messageInbox) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (inboxIsWithinTheirRange == true){ messageHashesInTheirRangeList = append(messageHashesInTheirRangeList, messageHash) } } sublistsToQueryList, err := helpers.SplitListIntoSublists(messageHashesInTheirRangeList, int(maximumMessageHashesPerRequest)) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } minimumRange, maximumRange := byteRange.GetMinimumMaximumInboxBounds() //Outputs: // -bool: Download successful // -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported Message Hash) // -error getDownloadedReportsInfoMap := func()(bool, map[[30]byte][26]byte, error){ retrievedReportsInfoMap := make(map[[30]byte][26]byte) for _, messageHashesSublist := range sublistsToQueryList{ downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetMessageReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, minimumRange, maximumRange, messageHashesSublist) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, end connection to host err := logger.AddLogEntry("Network", "Failed to download message reports info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reportHash, reportedHash := range currentRetrievedReportsInfoMap{ retrievedReportsInfoMap[reportHash] = reportedHash } } return true, retrievedReportsInfoMap, nil } downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } return true, connectionIdentifier, retrievedReportsInfoMap, minimumRange, maximumRange, nil } // We will retrieve reports based on message inbox range anyIntersectionFound, intersectionRangeStart, intersectionRangeEnd, err := byteRange.GetInboxIntersectionRangeFromTwoRanges(rangeToRetrieveStart, rangeToRetrieveEnd, theirInboxHostRangeStart, theirInboxHostRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (anyIntersectionFound == false){ // Host is not hosting any reports within the inboxes range that we are requesting // Skip this host return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } estimatedNumItemsInIntersectionRange, err := byteRange.GetEstimatedInboxSubrangeQuantity(theirInboxHostRangeStart, theirInboxHostRangeEnd, hostedMessagesQuantityInt64, intersectionRangeStart, intersectionRangeEnd) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } subrangesToQueryList, err := byteRange.SplitInboxRangeIntoEqualSubranges(intersectionRangeStart, intersectionRangeEnd, estimatedNumItemsInIntersectionRange, maximumMessageHashesPerRequest) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (hostProfileExists == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } if (connectionEstablished == false){ return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } //Outputs: // -bool: Download successful // -map[[30]byte][26]byte: Downloaded reports info map (Report Hash -> Reported message hash) // -error getDownloadedReportsInfoMap := func()(bool, map[[30]byte][26]byte, error){ retrievedReportsInfoMap := make(map[[30]byte][26]byte) for _, subrangeObject := range subrangesToQueryList{ subrangeStart := subrangeObject.SubrangeStart subrangeEnd := subrangeObject.SubrangeEnd emptyList := make([][26]byte, 0) downloadSuccessful, currentRetrievedReportsInfoMap, err := sendRequests.GetMessageReportsInfoFromHost(connectionIdentifier, hostIdentityHash, networkType, subrangeStart, subrangeEnd, emptyList) if (err != nil) { return false, nil, err } if (downloadSuccessful == false){ // Failed to download this sublist. // Could be that host went offline permanently or just this connection failed. // Either way, stop downloading from host err := logger.AddLogEntry("Network", "Failed to download message reports info from host.") if (err != nil) { return false, nil, err } return false, nil, nil } for reportHash, reportedHash := range currentRetrievedReportsInfoMap{ retrievedReportsInfoMap[reportHash] = reportedHash } } return true, retrievedReportsInfoMap, nil } downloadSuccessful, retrievedReportsInfoMap, err := getDownloadedReportsInfoMap() if (err != nil) { errB := peerClient.CloseConnection(connectionIdentifier) if (errB != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, errB } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } if (downloadSuccessful == false){ err := peerClient.CloseConnection(connectionIdentifier) if (err != nil) { return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, err } return false, [21]byte{}, nil, [10]byte{}, [10]byte{}, nil } return true, connectionIdentifier, retrievedReportsInfoMap, intersectionRangeStart, intersectionRangeEnd, nil } hostDownloadSuccessful, connectionIdentifier, retrievedReportsInfoMap, expectedRangeStart, expectedRangeEnd, err := getRetrievedReportsInfoMap() if (err != nil) { return false, err } if (hostDownloadSuccessful == false){ // We did not download any reports from this host. Skip them. return false, nil } defer peerClient.CloseConnection(connectionIdentifier) // Now we see if we have any of these reports, and if they do not fulfill the requests parameters // If they do not fulfill the request, we know host is malicious // We still download the reports to prevent host from knowing that we have those reports downloaded // We then mark the host as malicious reportHashesToDownloadList := make([][30]byte, 0) hostIsMalicious := false for receivedReportHash, receivedReportedHash := range retrievedReportsInfoMap{ //TODO: Disable this step if only 1 of Host/Moderator mode is enabled storedReportExists, storedReportBytes, err := badgerDatabase.GetReport(receivedReportHash) if (err != nil) { return false, err } checkIfHostIsOfferingInvalidReport := func()(bool, error){ if (storedReportExists == true){ ableToRead, storedReportHash, _, reportNetworkType, _, storedReportType, storedReportReportedHash, _, err := readReports.ReadReportAndHash(false, storedReportBytes) if (err != nil) { return false, err } if (ableToRead == false){ return false, errors.New("Database corrupt: Contains invalid report") } if (storedReportHash != receivedReportHash){ return false, errors.New("Database corrupt: Contains report with invalid entry key hash") } if (reportNetworkType != networkType){ return true, nil } areEqual := bytes.Equal(storedReportReportedHash, receivedReportedHash[:]) if (areEqual == false){ return true, nil } if (storedReportType != "Message"){ return true, nil } } // Now we see if we have the reviewed message metadata metadataExists, _, messageNetworkType, _, reportedMessageInbox, _, err := contentMetadata.GetMessageMetadata(receivedReportedHash) if (err != nil) { return false, err } if (metadataExists == false){ return false, nil } if (messageNetworkType != networkType){ // Report is reporting a message on a different networkType than the report // The host should have checked for this already // Host must be malicious return true, nil } inboxIsWithinRange, err := byteRange.CheckIfInboxIsWithinRange(expectedRangeStart, expectedRangeEnd, reportedMessageInbox) if (err != nil) { return false, err } if (inboxIsWithinRange == false){ return true, nil } return false, nil } hostIsOfferingInvalidReport, err := checkIfHostIsOfferingInvalidReport() if (err != nil) { return false, err } if (hostIsOfferingInvalidReport == true){ // Host is malicious, we must still download the report hostIsMalicious = true reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash) continue } if (storedReportExists == true){ // We already have the report, skip downloading it continue } // Report does not exist, we must check if we want to download it // This works differently for Hosting and Moderation // For instance, if the user is downloading reports for moderation, they will not redownload reports for messages that they have already reviewed shouldDownload, err := checkIfReportShouldBeDownloaded(receivedReportHash, receivedReportedHash) if (err != nil) { return false, err } if (shouldDownload == true){ reportHashesToDownloadList = append(reportHashesToDownloadList, receivedReportHash) continue } } if (len(reportHashesToDownloadList) == 0){ // Nothing to download, done with this host. if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } return true, nil } // Now we download reports // We convert the [26]byte value to []byte to supply to GetReportsFromHost reportsInfoMapForRequest := make(map[[30]byte][]byte) for reportHash, reportedMessageHash := range retrievedReportsInfoMap{ reportsInfoMapForRequest[reportHash] = reportedMessageHash[:] } maximumReportsInResponse := serverResponse.MaximumReportsInResponse_GetReports reportHashesSublistsList, err := helpers.SplitListIntoSublists(reportHashesToDownloadList, maximumReportsInResponse) if (err != nil) { return false, err } numberOfDownloadedReports := 0 for _, reportHashesSublist := range reportHashesSublistsList{ downloadSuccessful, reportsList, err := sendRequests.GetReportsFromHost(connectionIdentifier, hostIdentityHash, networkType, reportHashesSublist, reportsInfoMapForRequest) if (err != nil) { return false, err } if (downloadSuccessful == false){ // Request has failed // We will stop downloading from this host break } for _, reportBytes := range reportsList{ reportIsWellFormed, err := reportStorage.AddReport(reportBytes) if (err != nil) { return false, err } if (reportIsWellFormed == false){ return false, errors.New("GetReportsFromHost not verifying reports are well formed.") } } numberOfDownloadedReports += len(reportsList) } if (hostIsMalicious == true){ err := maliciousHosts.AddHostToMaliciousHostsList(hostIdentityHash) if (err != nil) { return false, err } } if (numberOfDownloadedReports != 0){ numberOfReportsDownloadedString := helpers.ConvertIntToString(numberOfDownloadedReports) err = logger.AddLogEntry("Network", "Successfully downloaded " + numberOfReportsDownloadedString + " reports from host.") if (err != nil) { return false, err } } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadReportsFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } //Inputs: // -bool: Allow clearnet // -[]string: List of identity hashes to retrieve viewable consensus statuses for // -map[[28]byte][16]byte: Map of Profile Hash -> Profile Identity Hash // Outputs: // -error func DownloadViewableStatusesFromHosts( allowClearnet bool, networkType byte, knownOrUnknown string, identityHashesToRetrieveList [][16]byte, profileHashesToRetrieveMap map[[28]byte][16]byte, numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadViewableStatusesFromHosts called with invalid networkType: " + networkTypeString) } if (knownOrUnknown != "Known" && knownOrUnknown != "Unknown"){ return errors.New("DownloadViewableStatusesFromHosts called with invalid knownOrUnknown: " + knownOrUnknown) } //Outputs: // -bool: Downloaded any statuses // -error downloadViewableStatusesFromHost := func(hostIdentityHash [16]byte)(bool, error){ exists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (exists == false){ // Host profile was deleted, skip host return false, nil } hostIsHostingMateProfiles, hostMateRangeStart, hostMateRangeEnd, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Mate") if (err != nil) { return false, err } hostIsHostingHostProfiles, _, _, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Host") if (err != nil) { return false, err } hostIsHostingModeratorProfiles, _, _, err := hostRanges.GetHostedIdentityHashRangeFromHostRawProfileMap(hostRawProfileMap, "Moderator") if (err != nil) { return false, err } checkIfHostIsHostingIdentityHash := func(identityHash [16]byte)(bool, error){ identityType, err := identity.GetIdentityTypeFromIdentityHash(identityHash) if (err != nil) { return false, err } if (identityType == "Mate"){ if (hostIsHostingMateProfiles == false){ return false, nil } isWithinTheirRange, err := byteRange.CheckIfIdentityHashIsWithinRange(hostMateRangeStart, hostMateRangeEnd, identityHash) if (err != nil) { return false, err } if (isWithinTheirRange == false){ return false, nil } return true, nil } if (identityType == "Host"){ if (hostIsHostingHostProfiles == false){ return false, nil } return true, nil } // identityType == "Moderator" if (hostIsHostingModeratorProfiles == false){ return false, nil } return true, nil } relevantIdentityHashesToRetrieveList := make([][16]byte, 0) for _, identityHash := range identityHashesToRetrieveList{ isHostingIdentity, err := checkIfHostIsHostingIdentityHash(identityHash) if (err != nil) { return false, err } if (isHostingIdentity == false){ continue } statusIsKnown, _, queriedHostsList, err := trustedViewableStatus.GetTrustedIdentityIsViewableStatus(identityHash, networkType) if (err != nil) { return false, err } if (knownOrUnknown == "Known"){ // We only want to retrieve known statuses. We dont need to avoid already queried hosts. if (statusIsKnown == false){ continue } } else { // We only want to retrieve unknown statuses if (statusIsKnown == true){ continue } // We must avoid hosts we have already queried alreadyQueried := slices.Contains(queriedHostsList, hostIdentityHash) if (alreadyQueried == true){ continue } } relevantIdentityHashesToRetrieveList = append(relevantIdentityHashesToRetrieveList, identityHash) } relevantProfileHashesToRetrieveList := make([][28]byte, 0) for profileHash, profileIdentityHash := range profileHashesToRetrieveMap{ isHostingIdentity, err := checkIfHostIsHostingIdentityHash(profileIdentityHash) if (err != nil) { return false, err } if (isHostingIdentity == false){ continue } statusIsKnown, _, queriedHostsList, err := trustedViewableStatus.GetTrustedProfileIsViewableStatus(profileHash) if (err != nil) { return false, err } if (knownOrUnknown == "Known"){ // We only want to retrieve known statuses. We dont need to avoid already queried hosts. if (statusIsKnown == false){ continue } } else { // We only want to retrieve unknown statuses if (statusIsKnown == true){ continue } // We must avoid hosts we have already queried alreadyQueried := slices.Contains(queriedHostsList, hostIdentityHash) if (alreadyQueried == true){ continue } } relevantProfileHashesToRetrieveList = append(relevantProfileHashesToRetrieveList, profileHash) } if (len(relevantIdentityHashesToRetrieveList) == 0 && len(relevantProfileHashesToRetrieveList) == 0){ // Nothing to retrieve from this host. Skip them. return false, nil } // Now we must split up our request to not exceed maximum size // The requests/responses for identity hashes will always be smaller than profile hashes, for the same number of identities/profiles // This is because profile hashes are larger than identity hashes // We will split request so that we can fit the maximum amount of identity/profile hashes in all of the requests numberOfIdentityHashesToRetrieve := len(relevantIdentityHashesToRetrieveList) numberOfProfileHashesToRetrieve := len(relevantProfileHashesToRetrieveList) maximumIdentityHashesInResponse := serverResponse.MaximumIdentitiesInResponse_GetIdentityViewableStatuses maximumProfileHashesInResponse := serverResponse.MaximumProfilesInResponse_GetProfileViewableStatuses numberOfRequestsNeededForIdentityHashes := float64(numberOfIdentityHashesToRetrieve)/float64(maximumIdentityHashesInResponse) numberOfRequestsNeededForProfileHashes := float64(numberOfProfileHashesToRetrieve)/float64(maximumProfileHashesInResponse) totalNumberOfRequestsNeeded, err := helpers.CeilFloat64ToInt64(numberOfRequestsNeededForIdentityHashes + numberOfRequestsNeededForProfileHashes) if (err != nil) { return false, err } numberOfIdentityHashesPerRequest := int64(numberOfIdentityHashesToRetrieve)/totalNumberOfRequestsNeeded numberOfProfileHashesPerRequest := int64(numberOfProfileHashesToRetrieve)/totalNumberOfRequestsNeeded identityHashesSublistsList, err := helpers.SplitListIntoSublists(relevantIdentityHashesToRetrieveList, int(numberOfIdentityHashesPerRequest)) if (err != nil) { return false, err } profileHashesSublistsList, err := helpers.SplitListIntoSublists(relevantProfileHashesToRetrieveList, int(numberOfProfileHashesPerRequest)) if (err != nil) { return false, err } identityHashesSublistsListLength := len(identityHashesSublistsList) profileHashesSublistsListLength := len(profileHashesSublistsList) getLongerListLength := func()int{ if (identityHashesSublistsListLength > profileHashesSublistsListLength){ return identityHashesSublistsListLength } return profileHashesSublistsListLength } longerListLength := getLongerListLength() totalDownloadedStatuses := 0 hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, err } if (hostProfileExists == false){ return false, nil } if (connectionEstablished == false){ return false, nil } defer peerClient.CloseConnection(connectionIdentifier) for i := 0; i < longerListLength; i++ { getIdentityHashesSublist := func()[][16]byte{ if (i < identityHashesSublistsListLength){ return identityHashesSublistsList[i] } emptyList := make([][16]byte, 0) return emptyList } getProfileHashesSublist := func()[][28]byte{ if (i < profileHashesSublistsListLength){ return profileHashesSublistsList[i] } emptyList := make([][28]byte, 0) return emptyList } identityHashesSublist := getIdentityHashesSublist() profileHashesSublist := getProfileHashesSublist() downloadSuccessful, identityHashStatusesMap, profileHashStatusesMap, err := sendRequests.GetViewableStatusesFromHost(connectionIdentifier, hostIdentityHash, networkType, identityHashesSublist, profileHashesSublist) if (err != nil) { return false, err } if (downloadSuccessful == false){ // This host has failed to respond. We will stop downloading from them. return false, nil } for identityHash, viewableStatus := range identityHashStatusesMap{ err := trustedViewableStatus.AddTrustedIdentityIsViewableStatus(identityHash, hostIdentityHash, networkType, viewableStatus) if (err != nil) { return false, err } totalDownloadedStatuses += 1 } for profileHash, viewableStatus := range profileHashStatusesMap{ err := trustedViewableStatus.AddTrustedProfileIsViewableStatus(profileHash, hostIdentityHash, viewableStatus) if (err != nil) { return false, err } profileIdentityHash, exists := profileHashesToRetrieveMap[profileHash] if (exists == false){ return false, errors.New("profileHashesToRetrieveMap missing profileHash") } if (viewableStatus == true){ // The host will only indicate a viewable profile if they know that the identity is not banned err := trustedViewableStatus.AddTrustedIdentityIsViewableStatus(profileIdentityHash, hostIdentityHash, networkType, true) if (err != nil) { return false, err } } totalDownloadedStatuses += 1 } } totalDownloadedStatusesString := helpers.ConvertIntToString(totalDownloadedStatuses) err = logger.AddLogEntry("Network", "Successfully downloaded " + totalDownloadedStatusesString + " viewable statuses from host.") if (err != nil) { return false, err } return true, nil } eligibleHostsList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil) { return err } queriedHosts := 0 for _, hostIdentityHash := range eligibleHostsList{ successfulDownload, err := downloadViewableStatusesFromHost(hostIdentityHash) if (err != nil) { return err } if (successfulDownload == true){ queriedHosts += 1 } if (queriedHosts >= numberOfHostsToQuery){ return nil } } return nil } func DownloadModeratorAddressDepositsFromHosts(allowClearnet bool, networkType byte, cryptocurrencyName string, knownOrUnknown string, numberOfHostsToQuery int)error{ isValid := helpers.VerifyNetworkType(networkType) if (isValid == false){ networkTypeString := helpers.ConvertByteToString(networkType) return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid networkType: " + networkTypeString) } if (cryptocurrencyName != "Ethereum" && cryptocurrencyName != "Cardano"){ return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid cryptocurrencyName: " + cryptocurrencyName) } if (knownOrUnknown != "Unknown" && knownOrUnknown != "Known"){ return errors.New("DownloadModeratorAddressDepositsFromHosts called with invalid knownOrUnknown: " + knownOrUnknown) } //TODO: Check if local Ethereum/Cardano node is being used (either here or in function that calls this function) //TODO: Check to see if less than 3 blockchain hosts exist, in which case, relax the required rule of 3 needed // We will first see which identity deposits are known/unknown //Outputs: // -bool: Downloaded any deposits // -error downloadDepositsFromHost := func(hostIdentityHash [16]byte)(bool, error){ profileExists, _, _, _, _, hostRawProfileMap, err := profileStorage.GetNewestUserProfile(hostIdentityHash, networkType) if (err != nil) { return false, err } if (profileExists == false){ return false, nil } exists, isHostingBlockchain, err := readProfiles.GetFormattedProfileAttributeFromRawProfileMap(hostRawProfileMap, "Hosting" + cryptocurrencyName + "Blockchain") if (err != nil) { return false, err } if (exists == false || isHostingBlockchain != "Yes"){ return false, nil } localBlockchainNodeEnabled, addressesToUpdateList, err := moderatorScores.GetModeratorCryptoDepositAddressesList(cryptocurrencyName, knownOrUnknown) if (err != nil){ return false, err } if (localBlockchainNodeEnabled == true){ // We must have enabled the node after our earlier check // Skip all remaining hosts. return false, nil } if (len(addressesToUpdateList) == 0){ // No addresses to update remain. return false, nil } //TODO: Add limit to size of below hostProfileExists, connectionEstablished, connectionIdentifier, err := peerClient.EstablishNewConnectionToHost(allowClearnet, hostIdentityHash, networkType) if (err != nil){ return false, err } if (hostProfileExists == false){ return false, nil } if (connectionEstablished == false){ return false, nil } defer peerClient.CloseConnection(connectionIdentifier) successfulDownload, retrievedDepositObjectsList, addressesWithNoDepositsList, err := sendRequests.GetAddressDepositsFromHost(connectionIdentifier, hostIdentityHash, networkType, cryptocurrencyName, addressesToUpdateList) if (err != nil) { return false, err } if (successfulDownload == false){ err := logger.AddLogEntry("Network", "Failed to retrieve address deposits from host.") if (err != nil) { return false, err } // Skip to next host return false, nil } err = trustedAddressDeposits.AddAddressDepositObjectsListToCache(hostIdentityHash, cryptocurrencyName, retrievedDepositObjectsList) if (err != nil) { return false, err } err = trustedAddressDeposits.AddAddressesWithNoDepositsToCache(hostIdentityHash, cryptocurrencyName, addressesWithNoDepositsList) if (err != nil) { return false, err } return true, nil } // We query each host for the identity deposits we want // After each host, we get the new known/unknown identities list to make // sure we do not query deposits which were unknown but have become known hostsToQueryList, err := eligibleHosts.GetEligibleHostsList(networkType) if (err != nil){ return err } hostsQueried := 0 for _, hostIdentityHash := range hostsToQueryList{ downloadedAnyDeposits, err := downloadDepositsFromHost(hostIdentityHash) if (err != nil) { return err } if (downloadedAnyDeposits == true){ hostsQueried += 1 } if (hostsQueried >= numberOfHostsToQuery){ break } } return nil }