// createGeneticModels.go provides an interface to create genetic prediction models // These are neural networks which predict traits such as eye color from raw genome files // The OpenSNP.org dataset is used, and more datasets will be added in the future. // You must download the dataset and extract it. The instructions are described in the utility. // The trained models are saved in the /resources/geneticPredictionModels package for use in the Seekia app. package main import "fyne.io/fyne/v2" import "fyne.io/fyne/v2/app" import "fyne.io/fyne/v2/widget" import "fyne.io/fyne/v2/container" import "fyne.io/fyne/v2/theme" import "fyne.io/fyne/v2/layout" import "fyne.io/fyne/v2/dialog" import "fyne.io/fyne/v2/data/binding" import "seekia/resources/geneticReferences/traits" import "seekia/resources/geneticReferences/locusMetadata" import "seekia/internal/encoding" import "seekia/internal/genetics/locusValue" import "seekia/internal/genetics/prepareRawGenomes" import "seekia/internal/genetics/readRawGenomes" import "seekia/internal/genetics/geneticPrediction" import "seekia/internal/helpers" import "seekia/internal/imagery" import "seekia/internal/localFilesystem" import "seekia/internal/genetics/readBiobankData" import "errors" import "crypto/sha256" import "bytes" import "image/color" import "io" import "os" import "strings" import "sync" import "slices" import "math" import mathRand "math/rand/v2" import goFilepath "path/filepath" import "time" func main(){ err := traits.InitializeTraitVariables() if (err != nil){ panic(err) return } app := app.New() customTheme := getCustomFyneTheme() app.Settings().SetTheme(customTheme) window := app.NewWindow("Seekia - Create Genetic Models Utility") windowSize := fyne.NewSize(600, 600) window.Resize(windowSize) window.CenterOnScreen() setHomePage(window) window.ShowAndRun() } func getWidgetCentered(widget fyne.Widget)*fyne.Container{ widgetCentered := container.NewHBox(layout.NewSpacer(), widget, layout.NewSpacer()) return widgetCentered } func getLabelCentered(text string) *fyne.Container{ label := widget.NewLabel(text) labelCentered := container.NewHBox(layout.NewSpacer(), label, layout.NewSpacer()) return labelCentered } func getBoldLabel(text string) fyne.Widget{ titleStyle := fyne.TextStyle{ Bold: true, Italic: false, Monospace: false, } boldLabel := widget.NewLabelWithStyle(text, fyne.TextAlign(fyne.TextAlignCenter), titleStyle) return boldLabel } func getItalicLabel(text string) fyne.Widget{ italicTextStyle := fyne.TextStyle{ Bold: false, Italic: true, Monospace: false, } italicLabel := widget.NewLabelWithStyle(text, fyne.TextAlign(fyne.TextAlignCenter), italicTextStyle) return italicLabel } func getBoldLabelCentered(inputText string)*fyne.Container{ boldLabel := getBoldLabel(inputText) boldLabelCentered := container.NewHBox(layout.NewSpacer(), boldLabel, layout.NewSpacer()) return boldLabelCentered } func getItalicLabelCentered(inputText string)*fyne.Container{ italicLabel := getItalicLabel(inputText) italicLabelCentered := container.NewHBox(layout.NewSpacer(), italicLabel, layout.NewSpacer()) return italicLabelCentered } func showUnderConstructionDialog(window fyne.Window){ dialogTitle := "Under Construction" dialogMessageA := getLabelCentered("Seekia is under construction.") dialogMessageB := getLabelCentered("This page/feature needs to be built.") dialogContent := container.NewVBox(dialogMessageA, dialogMessageB) dialog.ShowCustom(dialogTitle, "Close", dialogContent, window) } func getBackButtonCentered(previousPage func())*fyne.Container{ backButton := getWidgetCentered(widget.NewButtonWithIcon("Go Back", theme.NavigateBackIcon(), previousPage)) return backButton } func setErrorEncounteredPage(window fyne.Window, err error, previousPage func()){ title := getBoldLabelCentered("Error Encountered") backButton := getBackButtonCentered(previousPage) description1 := getLabelCentered("Something went wrong. Report this error to Seekia developers.") header := container.NewVBox(title, backButton, widget.NewSeparator(), description1, widget.NewSeparator()) getErrorString := func()string{ if (err == nil){ return "No nav bar error encountered page called with nil error." } errorString := err.Error() return errorString } errorString := getErrorString() errorLabel := widget.NewLabel(errorString) errorLabel.Wrapping = 3 errorLabel.Alignment = 1 errorLabel.TextStyle = fyne.TextStyle{ Bold: true, Italic: false, Monospace: false, } //TODO: Add copyable toggle page := container.NewBorder(header, nil, nil, nil, errorLabel) window.SetContent(page) } // This loading screen shows no progress, so it should only be used when retrieving progress is impossible func setLoadingScreen(window fyne.Window, pageTitle string, loadingText string){ title := getBoldLabelCentered(pageTitle) loadingLabel := getWidgetCentered(getItalicLabel(loadingText)) progressBar := getWidgetCentered(widget.NewProgressBarInfinite()) pageContent := container.NewVBox(title, loadingLabel, progressBar) page := container.NewCenter(pageContent) window.SetContent(page) } func setHomePage(window fyne.Window){ currentPage := func(){setHomePage(window)} title := getBoldLabelCentered("Create Genetic Models Utility") description1 := getLabelCentered("This utility is used to create the genetic prediction models.") description2 := getLabelCentered("These models are used to predict traits such as eye color from raw genome files.") description3 := getLabelCentered("Seekia aims to have open source and reproducible genetic prediction technology.") step1Label := getLabelCentered("Step 1:") downloadTrainingDataButton := getWidgetCentered(widget.NewButton("Download Training Data", func(){ setDownloadTrainingDataPage(window, currentPage) })) step2Label := getLabelCentered("Step 2:") extractTrainingDataButton := getWidgetCentered(widget.NewButton("Extract Training Data", func(){ setExtractTrainingDataPage(window, currentPage) })) step3Label := getLabelCentered("Step 3:") createTrainingDataButton := getWidgetCentered(widget.NewButton("Create Training Data", func(){ setCreateTrainingDataPage(window, currentPage) })) step4Label := getLabelCentered("Step 4:") trainModelsButton := getWidgetCentered(widget.NewButton("Train Models", func(){ setTrainModelsPage(window, currentPage) })) step5Label := getLabelCentered("Step 5:") testModelsButton := getWidgetCentered(widget.NewButton("Test Models", func(){ setTestModelsPage(window, currentPage) })) //TODO: A page to verify the checksums of the generated .gob models page := container.NewVBox(title, widget.NewSeparator(), description1, description2, description3, widget.NewSeparator(), step1Label, downloadTrainingDataButton, widget.NewSeparator(), step2Label, extractTrainingDataButton, widget.NewSeparator(), step3Label, createTrainingDataButton, widget.NewSeparator(), step4Label, trainModelsButton, widget.NewSeparator(), step5Label, testModelsButton) window.SetContent(page) } func setDownloadTrainingDataPage(window fyne.Window, previousPage func()){ currentPage := func(){setDownloadTrainingDataPage(window, previousPage)} title := getBoldLabelCentered("Download Training Data") backButton := getBackButtonCentered(previousPage) description1 := getLabelCentered("You must download the OpenSNP.org data dump file.") description2 := getLabelCentered("This is a .tar.gz file which was created in August of 2023.") description3 := getLabelCentered("It will be hosted on IPFS, a decentralized data sharing network.") description4 := getLabelCentered("You must use an IPFS client to download the file.") description5 := getLabelCentered("You can also download it via a torrent or web server if someone shares it elsewhere.") currentClipboard := window.Clipboard() ipfsIdentifierTitle := getLabelCentered("IPFS Content Identifier:") ipfsIdentifierLabel := getBoldLabelCentered("Qme64v7Go941s3psokZ7aDngQR6Tdv55jDhUDdLZXsRiRh") ipfsIdentifierCopyToClipboardButton := getWidgetCentered(widget.NewButtonWithIcon("Copy", theme.ContentCopyIcon(), func(){ currentClipboard.SetContent("Qme64v7Go941s3psokZ7aDngQR6Tdv55jDhUDdLZXsRiRh") })) fileNameTitle := getLabelCentered("File Name:") fileNameLabel := getBoldLabelCentered("OpenSNPDataArchive.tar.gz") fileHashTitle := getLabelCentered("File SHA256 Checksum Hash:") fileHashLabel := getBoldLabelCentered("49f84fb71cb12df718a80c1ce25f6370ba758cbee8f24bd8a6d4f0da2e3c51ee") fileSizeTitle := getLabelCentered("File Size:") fileSizeLabel := getBoldLabelCentered("48,961,240 bytes (50.1 GB)") fileExtractedSizeTitle := getLabelCentered("File Extracted Size:") fileExtractedSizeLabel := getBoldLabelCentered("128,533,341,751 bytes (119.7 GB)") verifyFileTitle := getBoldLabelCentered("Verify File") verifyFileDescription1 := getLabelCentered("You can use the Seekia client to verify your downloaded file.") verifyFileDescription2 := getLabelCentered("Press the button below and select your file.") verifyFileDescription3 := getLabelCentered("This will take a while, because the file contents must be hashed.") selectFileCallbackFunction := func(fyneFileObject fyne.URIReadCloser, err error){ if (err != nil){ setErrorEncounteredPage(window, err, currentPage) return } if (fyneFileObject == nil){ return } setLoadingScreen(window, "Hashing File", "Calculating file hash...") filePath := fyneFileObject.URI().String() filePath = strings.TrimPrefix(filePath, "file://") fileObject, err := os.Open(filePath) if (err != nil){ setErrorEncounteredPage(window, err, currentPage) return } defer fileObject.Close() //TODO: Use Blake3 instead of sha256 for faster hashing hasher := sha256.New() _, err = io.Copy(hasher, fileObject) if (err != nil){ setErrorEncounteredPage(window, err, currentPage) return } hashResultBytes := hasher.Sum(nil) expectedResult := "49f84fb71cb12df718a80c1ce25f6370ba758cbee8f24bd8a6d4f0da2e3c51ee" expectedResultBytes, err := encoding.DecodeHexStringToBytes(expectedResult) if (err != nil){ setErrorEncounteredPage(window, err, currentPage) return } currentPage() bytesAreEqual := bytes.Equal(hashResultBytes, expectedResultBytes) if (bytesAreEqual == false){ title := "File Is Invalid" dialogMessage1 := getLabelCentered("The file you downloaded is not valid.") dialogMessage2 := getLabelCentered("The SHA256 Checksum does not match the expected checksum.") dialogContent := container.NewVBox(dialogMessage1, dialogMessage2) dialog.ShowCustom(title, "Close", dialogContent, window) } else { title := "File Is Valid" dialogMessage1 := getLabelCentered("The file you downloaded is valid!") dialogMessage2 := getLabelCentered("The SHA256 Checksum matches the expected checksum.") dialogContent := container.NewVBox(dialogMessage1, dialogMessage2) dialog.ShowCustom(title, "Close", dialogContent, window) } } verifyFileButton := getWidgetCentered(widget.NewButton("Verify File", func(){ dialog.ShowFileOpen(selectFileCallbackFunction, window) })) page := container.NewVBox(title, backButton, widget.NewSeparator(), description1, description2, description3, description4, description5, widget.NewSeparator(), ipfsIdentifierTitle, ipfsIdentifierLabel, ipfsIdentifierCopyToClipboardButton, widget.NewSeparator(), fileNameTitle, fileNameLabel, widget.NewSeparator(), fileHashTitle, fileHashLabel, widget.NewSeparator(), fileSizeTitle, fileSizeLabel, widget.NewSeparator(), fileExtractedSizeTitle, fileExtractedSizeLabel, widget.NewSeparator(), verifyFileTitle, verifyFileDescription1, verifyFileDescription2, verifyFileDescription3, verifyFileButton) scrollablePage := container.NewVScroll(page) window.SetContent(scrollablePage) } func setExtractTrainingDataPage(window fyne.Window, previousPage func()){ currentPage := func(){setExtractTrainingDataPage(window, previousPage)} title := getBoldLabelCentered("Extract Training Data") backButton := getBackButtonCentered(previousPage) description1 := getLabelCentered("You must extract the downloaded OpenSNPDataArchive.tar.gz to a folder.") description2 := getLabelCentered("Once you have extracted the file, select the extracted folder using the page below.") currentLocationTitle := getLabelCentered("Current Folder Location:") getCurrentLocationLabel := func()(*fyne.Container, error){ fileExists, fileContents, err := localFilesystem.GetFileContents("./OpenSNPDataArchiveFolderpath.txt") if (err != nil) { return nil, err } if (fileExists == false){ noneLabel := getItalicLabelCentered("None") return noneLabel, nil } folderpathLabel := getBoldLabelCentered(string(fileContents)) return folderpathLabel, nil } currentLocationLabel, err := getCurrentLocationLabel() if (err != nil) { setErrorEncounteredPage(window, err, previousPage) return } selectFolderCallbackFunction := func(folderObject fyne.ListableURI, err error){ if (err != nil){ title := "Failed to open folder." dialogMessage := getLabelCentered("Report this error to Seekia developers: " + err.Error()) dialogContent := container.NewVBox(dialogMessage) dialog.ShowCustom(title, "Close", dialogContent, window) return } if (folderObject == nil) { return } folderPath := folderObject.Path() fileContents := []byte(folderPath) err = localFilesystem.CreateOrOverwriteFile(fileContents, "./", "OpenSNPDataArchiveFolderpath.txt") if (err != nil){ title := "Failed to save file." dialogMessage := getLabelCentered("Report this error to Seekia developers: " + err.Error()) dialogContent := container.NewVBox(dialogMessage) dialog.ShowCustom(title, "Close", dialogContent, window) return } currentPage() } selectFolderLocationButton := getWidgetCentered(widget.NewButtonWithIcon("Select Folder Location", theme.FolderIcon(), func(){ dialog.ShowFolderOpen(selectFolderCallbackFunction, window) })) page := container.NewVBox(title, backButton, widget.NewSeparator(), description1, description2, widget.NewSeparator(), currentLocationTitle, currentLocationLabel, widget.NewSeparator(), selectFolderLocationButton) scrollablePage := container.NewVScroll(page) window.SetContent(scrollablePage) } func setCreateTrainingDataPage(window fyne.Window, previousPage func()){ currentPage := func(){setCreateTrainingDataPage(window, previousPage)} title := getBoldLabelCentered("Create Training Data") backButton := getBackButtonCentered(previousPage) description1 := getLabelCentered("Press the button below to begin creating the training data.") description2 := getLabelCentered("This will prepare each user's genome into a file to use to train each neural network.") description3 := getLabelCentered("This will take a while.") beginCreatingButton := getWidgetCentered(widget.NewButtonWithIcon("Begin Creating Data", theme.MediaPlayIcon(), func(){ setStartAndMonitorCreateTrainingDataPage(window, currentPage) })) page := container.NewVBox(title, backButton, widget.NewSeparator(), description1, description2, description3, beginCreatingButton) window.SetContent(page) } func setStartAndMonitorCreateTrainingDataPage(window fyne.Window, previousPage func()){ err := locusMetadata.InitializeLocusMetadataVariables() if (err != nil){ setErrorEncounteredPage(window, err, previousPage) return } title := getBoldLabelCentered("Creating Training Data") fileExists, fileContents, err := localFilesystem.GetFileContents("./OpenSNPDataArchiveFolderpath.txt") if (err != nil) { setErrorEncounteredPage(window, err, previousPage) return } if (fileExists == false){ backButton := getBackButtonCentered(previousPage) description1 := getBoldLabelCentered("You have not selected your OpenSNP data archive folderpath.") description2 := getLabelCentered("Go back to step 2 and follow the instructions.") page := container.NewVBox(title, backButton, widget.NewSeparator(), description1, description2) window.SetContent(page) return } dataArchiveFolderpath := string(fileContents) progressDetailsBinding := binding.NewString() estimatedTimeRemainingBinding := binding.NewString() progressPercentageBinding := binding.NewFloat() loadingBar := getWidgetCentered(widget.NewProgressBarWithData(progressPercentageBinding)) progressDetailsTitle := getBoldLabelCentered("Progress Details:") progressDetailsLabel := widget.NewLabelWithData(progressDetailsBinding) progressDetailsLabel.TextStyle = fyne.TextStyle{ Bold: false, Italic: true, Monospace: false, } progressDetailsLabelCentered := getWidgetCentered(progressDetailsLabel) estimatedTimeRemainingLabel := widget.NewLabelWithData(estimatedTimeRemainingBinding) estimatedTimeRemainingLabel.TextStyle = fyne.TextStyle{ Bold: false, Italic: true, Monospace: false, } estimatedTimeRemainingLabelCentered := getWidgetCentered(estimatedTimeRemainingLabel) // We set this bool to true to stop the createData process var createDataIsStoppedBoolMutex sync.RWMutex createDataIsStoppedBool := false cancelButton := getWidgetCentered(widget.NewButtonWithIcon("Cancel", theme.CancelIcon(), func(){ createDataIsStoppedBoolMutex.Lock() createDataIsStoppedBool = true createDataIsStoppedBoolMutex.Unlock() previousPage() })) page := container.NewVBox(title, widget.NewSeparator(), loadingBar, progressDetailsTitle, progressDetailsLabelCentered, estimatedTimeRemainingLabelCentered, widget.NewSeparator(), cancelButton) window.SetContent(page) createTrainingDataFunction := func(){ var processProgressMutex sync.RWMutex // This stores the progress of creating the training data (0-1) processProgress := float64(0) startUpdateTimeRemainingDisplayFunction := func(){ // This function updates the estimated time remaining label binding updateTimeRemainingDisplayFunction := func()error{ startTime := time.Now().Unix() for{ createDataIsStoppedBoolMutex.RLock() createDataIsStopped := createDataIsStoppedBool createDataIsStoppedBoolMutex.RUnlock() if (createDataIsStopped == true){ // User exited the process/Process has completed return nil } processProgressMutex.RLock() currentProcessProgress := processProgress processProgressMutex.RUnlock() if (currentProcessProgress == 0){ estimatedTimeRemainingBinding.Set("Calculating required time...") time.Sleep(time.Second) continue } // We calculate how long we think it will take for the process to complete currentTime := time.Now().Unix() secondsElapsed := currentTime - startTime // processProgress is a float64 which stores the progress as a value between 0-1 // To get the estimated total time the process will take, we divide the seconds elapsed by the proportion of progress // For example: // 0.1 (10%) at 10 seconds == Total process will take 100 seconds // 0.5 (50%) at 20 seconds == Total process will take 40 seconds totalSeconds := float64(secondsElapsed)/currentProcessProgress estimatedSecondsRemaining := int64(totalSeconds) - secondsElapsed estimatedTimeRemainingTranslated, err := helpers.ConvertUnixTimeDurationToUnitsTimeTranslated(estimatedSecondsRemaining, false) if (err != nil) { return err } estimatedTimeRemainingBinding.Set("Estimated Time Remaining: " + estimatedTimeRemainingTranslated) time.Sleep(time.Second) } // This should never be reached return errors.New("updateTimeRemainingDisplayFunction loop has broken.") } err := updateTimeRemainingDisplayFunction() if (err != nil){ createDataIsStoppedBoolMutex.Lock() createDataIsStoppedBool = true createDataIsStoppedBoolMutex.Unlock() setErrorEncounteredPage(window, err, previousPage) return } } go startUpdateTimeRemainingDisplayFunction() //Outputs: // -bool: Process completed (true == was not stopped mid-way) // -bool: Data archive is well formed // -error createTrainingData := func()(bool, bool, error){ phenotypesFilepath := goFilepath.Join(dataArchiveFolderpath, "OpenSNPData", "phenotypes_202308230100.csv") fileObject, err := os.Open(phenotypesFilepath) if (err != nil){ fileDoesNotExist := os.IsNotExist(err) if (fileDoesNotExist == true){ // Archive is corrupt return true, false, nil } return false, false, err } defer fileObject.Close() fileIsWellFormed, userPhenotypesList_OpenSNP := readBiobankData.ReadOpenSNPPhenotypesFile(fileObject) if (fileIsWellFormed == false){ // Archive is corrupt return true, false, nil } // This is the folderpath for the folder which contains all of the user raw genomes openSNPRawGenomesFolderpath := goFilepath.Join(dataArchiveFolderpath, "OpenSNPData") filesList, err := os.ReadDir(openSNPRawGenomesFolderpath) if (err != nil) { return false, false, err } // Map Structure: User ID -> List of user raw genome filepaths userRawGenomeFilepathsMap := make(map[int][]string) for _, filesystemObject := range filesList{ filepathIsFolder := filesystemObject.IsDir() if (filepathIsFolder == true){ // Archive is corrupt return true, false, nil } fileName := filesystemObject.Name() // Example of a raw genome filename: "user1_file9_yearofbirth_1985_sex_XY.23andme" userIDWithRawGenomeInfo, fileIsUserGenome := strings.CutPrefix(fileName, "user") if (fileIsUserGenome == false){ // File is not a user genome, skip it. continue } userIDString, rawGenomeInfo, separatorFound := strings.Cut(userIDWithRawGenomeInfo, "_") if (separatorFound == false){ // Archive is corrupt return true, false, nil } userID, err := helpers.ConvertStringToInt(userIDString) if (err != nil){ // Archive is corrupt return true, false, nil } getFileIsReadableStatus := func()bool{ is23andMe := strings.HasSuffix(rawGenomeInfo, ".23andme.txt") if (is23andMe == true){ // We can read this file return true } isAncestry := strings.HasSuffix(rawGenomeInfo, ".ancestry.txt") if (isAncestry == true){ // We can read this file return true } // We cannot read this raw genome file //TODO: Add ability to read more raw genome files return false } fileIsReadable := getFileIsReadableStatus() if (fileIsReadable == true){ rawGenomeFilepath := goFilepath.Join(openSNPRawGenomesFolderpath, fileName) existingList, exists := userRawGenomeFilepathsMap[userID] if (exists == false){ userRawGenomeFilepathsMap[userID] = []string{rawGenomeFilepath} } else { existingList = append(existingList, rawGenomeFilepath) userRawGenomeFilepathsMap[userID] = existingList } } } // We create the folder to store the training data _, err = localFilesystem.CreateFolder("./TrainingData") if (err != nil) { return false, false, err } //TODO: Add more traits traitNamesList := []string{"Eye Color", "Lactose Tolerance", "Height"} // We create the folders for each trait's training data for _, traitName := range traitNamesList{ traitNameWithoutWhitespace := strings.ReplaceAll(traitName, " ", "") folderpath := goFilepath.Join("./TrainingData/", traitNameWithoutWhitespace) _, err = localFilesystem.CreateFolder(folderpath) if (err != nil) { return false, false, err } folderExists, err := localFilesystem.DeleteAllFolderContents(folderpath) if (err != nil) { return false, false, err } if (folderExists == false){ return false, false, errors.New("CreateFolder failed to create folder.") } } numberOfUserPhenotypeDataObjects := len(userPhenotypesList_OpenSNP) maximumIndex := numberOfUserPhenotypeDataObjects-1 numberOfUsersString := helpers.ConvertIntToString(numberOfUserPhenotypeDataObjects) for index, userPhenotypeDataObject := range userPhenotypesList_OpenSNP{ createDataIsStoppedBoolMutex.RLock() createDataIsStopped := createDataIsStoppedBool createDataIsStoppedBoolMutex.RUnlock() if (createDataIsStopped == true){ // User exited the process return false, false, nil } userIndexString := helpers.ConvertIntToString(index + 1) progressDetailsStatus := "Processing User " + userIndexString + "/" + numberOfUsersString err = progressDetailsBinding.Set(progressDetailsStatus) if (err != nil) { return false, false, err } trainingProgressPercentage, err := helpers.ScaleIntProportionally(true, index, 0, maximumIndex, 0, 100) if (err != nil) { return false, false, err } trainingProgressFloat64 := float64(trainingProgressPercentage)/100 err = progressPercentageBinding.Set(trainingProgressFloat64) if (err != nil) { return false, false, err } processProgressMutex.Lock() processProgress = trainingProgressFloat64 processProgressMutex.Unlock() userID := userPhenotypeDataObject.UserID userRawGenomeFilepathsList, exists := userRawGenomeFilepathsMap[userID] if (exists == false){ // User has no genomes continue } // We read all of the user's raw genomes and combine them into a single genomeMap which excludes conflicting loci values userRawGenomesWithMetadataList := make([]prepareRawGenomes.RawGenomeWithMetadata, 0) for _, userRawGenomeFilepath := range userRawGenomeFilepathsList{ //Outputs: // -bool: Able to read raw genome file // -bool: Genome is phased // -map[int64]readRawGenomes.RawGenomeLocusValue // -error readRawGenomeMap := func()(bool, bool, map[int64]readRawGenomes.RawGenomeLocusValue, error){ fileObject, err := os.Open(userRawGenomeFilepath) if (err != nil) { return false, false, nil, err } defer fileObject.Close() _, _, _, _, genomeIsPhased, rawGenomeMap, err := readRawGenomes.ReadRawGenomeFile(fileObject) if (err != nil) { //log.Println("Raw genome file is malformed: " + userRawGenomeFilepath + ". Reason: " + err.Error()) return false, false, nil, nil } return true, genomeIsPhased, rawGenomeMap, nil } ableToReadRawGenome, rawGenomeIsPhased, rawGenomeMap, err := readRawGenomeMap() if (err != nil){ return false, false, err } if (ableToReadRawGenome == false){ // We cannot read this genome file // Many of the genome files are unreadable. //TODO: Improve ability to read slightly corrupted genome files continue } newGenomeIdentifier, err := helpers.GetNewRandom16ByteArray() if (err != nil) { return false, false, err } rawGenomeWithMetadata := prepareRawGenomes.RawGenomeWithMetadata{ GenomeIdentifier: newGenomeIdentifier, GenomeIsPhased: rawGenomeIsPhased, RawGenomeMap: rawGenomeMap, } userRawGenomesWithMetadataList = append(userRawGenomesWithMetadataList, rawGenomeWithMetadata) } if (len(userRawGenomesWithMetadataList) == 0){ // None of the user's genome files are readable continue } //Outputs: // -bool: Any useful locations exist in any of the user's genomes // -map[int64]locusValue.LocusValue // -error getUserLociValuesMap := func()(bool, map[int64]locusValue.LocusValue, error){ updatePercentageCompleteFunction := func(_ int)error{ return nil } anyUsefulLocationsExist, genomesWithMetadataList, _, combinedGenomesExist, onlyExcludeConflictsGenomeIdentifier, _, err := prepareRawGenomes.GetGenomesWithMetadataListFromRawGenomesList(userRawGenomesWithMetadataList, updatePercentageCompleteFunction) if (err != nil) { return false, nil, err } if (anyUsefulLocationsExist == false){ // None of the user's genomes have any useful locations return false, nil, nil } if (combinedGenomesExist == false){ if (len(genomesWithMetadataList) != 1){ return false, nil, errors.New("GetGenomesWithMetadataListFromRawGenomesList returning non-1 length genomesWithMetadataList when combinedGenomesExist == false") } // Only 1 genome exists genomeWithMetadataObject := genomesWithMetadataList[0] genomeMap := genomeWithMetadataObject.GenomeMap return true, genomeMap, nil } for _, genomeWithMetadataObject := range genomesWithMetadataList{ genomeIdentifier := genomeWithMetadataObject.GenomeIdentifier if (genomeIdentifier == onlyExcludeConflictsGenomeIdentifier){ genomeMap := genomeWithMetadataObject.GenomeMap return true, genomeMap, nil } } return false, nil, errors.New("OnlyExcludeConflicts genome not found from GetGenomesWithMetadataListFromRawGenomesList's returned list.") } anyUsefulLocationsExist, userLociValuesMap, err := getUserLociValuesMap() if (err != nil) { return false, false, err } if (anyUsefulLocationsExist == false){ // None of the user's genome files contain any useful locations continue } for _, traitName := range traitNamesList{ traitNameWithoutWhitespace := strings.ReplaceAll(traitName, " ", "") trainingDataFolderpath := goFilepath.Join("./TrainingData", traitNameWithoutWhitespace) userDataExists, userTrainingDataList, err := geneticPrediction.CreateGeneticPredictionTrainingData_OpenSNP(traitName, userPhenotypeDataObject, userLociValuesMap) if (err != nil) { return false, false, err } if (userDataExists == false){ // User cannot be used for training continue } for index, trainingData := range userTrainingDataList{ userTrainingDataBytes, err := geneticPrediction.EncodeTrainingDataObjectToBytes(trainingData) if (err != nil) { return false, false, err } trainingDataIndexString := helpers.ConvertIntToString(index+1) userIDString := helpers.ConvertIntToString(userID) trainingDataFilename := "User" + userIDString + "_TrainingData_" + trainingDataIndexString + ".gob" err = localFilesystem.CreateOrOverwriteFile(userTrainingDataBytes, trainingDataFolderpath, trainingDataFilename) if (err != nil) { return false, false, err } } } } createDataIsStoppedBoolMutex.Lock() createDataIsStoppedBool = true createDataIsStoppedBoolMutex.Unlock() return true, true, nil } processIsComplete, archiveIsWellFormed, err := createTrainingData() if (err != nil){ createDataIsStoppedBoolMutex.Lock() createDataIsStoppedBool = true createDataIsStoppedBoolMutex.Unlock() setErrorEncounteredPage(window, err, previousPage) return } if (processIsComplete == false){ // User exited the page return } if (archiveIsWellFormed == false){ title := getBoldLabelCentered("OpenSNP Archive Is Corrupt") description1 := getBoldLabelCentered("Your downloaded OpenSNP data archive is corrupt.") description2 := getLabelCentered("The extracted folder contents do not match what the archive should contain.") description3 := getLabelCentered("You should re-extract the contents of the archive.") exitButton := getWidgetCentered(widget.NewButtonWithIcon("Exit", theme.CancelIcon(), previousPage)) page := container.NewVBox(title, widget.NewSeparator(), description1, description2, description3, exitButton) window.SetContent(page) return } setCreateTrainingDataIsCompletePage(window) } go createTrainingDataFunction() } func setCreateTrainingDataIsCompletePage(window fyne.Window){ title := getBoldLabelCentered("Creating Data Is Complete") description1 := getLabelCentered("Creating training data is complete!") description2 := getLabelCentered("The data have been saved in the TrainingData folder.") exitButton := getWidgetCentered(widget.NewButtonWithIcon("Exit", theme.CancelIcon(), func(){ setHomePage(window) })) page := container.NewVBox(title, widget.NewSeparator(), description1, description2, exitButton) window.SetContent(page) } func setTrainModelsPage(window fyne.Window, previousPage func()){ currentPage := func(){setTrainModelsPage(window, previousPage)} title := getBoldLabelCentered("Train Models") backButton := getBackButtonCentered(previousPage) description1 := getLabelCentered("Press the button below to begin training a genetic model.") description2 := getLabelCentered("This will train a neural network using the user training data.") description3 := getLabelCentered("This will take a while.") description4 := getLabelCentered("You must select a trait model to train.") traitNamesList := []string{"Eye Color", "Lactose Tolerance", "Height"} traitNameSelector := widget.NewSelect(traitNamesList, nil) beginTrainingButton := getWidgetCentered(widget.NewButtonWithIcon("Begin Training Model", theme.MediaPlayIcon(), func(){ selectedTraitIndex := traitNameSelector.SelectedIndex() if (selectedTraitIndex < 0){ title := "No Trait Selected" dialogMessage1 := getLabelCentered("You must select a trait model to train.") dialogContent := container.NewVBox(dialogMessage1) dialog.ShowCustom(title, "Close", dialogContent, window) return } traitName := traitNameSelector.Selected setStartAndMonitorTrainModelPage(window, traitName, currentPage) })) traitNameSelectorCentered := getWidgetCentered(traitNameSelector) page := container.NewVBox(title, backButton, widget.NewSeparator(), description1, description2, description3, description4, widget.NewSeparator(), traitNameSelectorCentered, widget.NewSeparator(), beginTrainingButton) window.SetContent(page) } func setStartAndMonitorTrainModelPage(window fyne.Window, traitName string, previousPage func()){ title := getBoldLabelCentered("Train Model") //TODO: Verify TrainingData folder integrity progressDetailsBinding := binding.NewString() estimatedTimeRemainingBinding := binding.NewString() progressPercentageBinding := binding.NewFloat() loadingBar := getWidgetCentered(widget.NewProgressBarWithData(progressPercentageBinding)) progressDetailsTitle := getBoldLabelCentered("Progress Details:") progressDetailsLabel := widget.NewLabelWithData(progressDetailsBinding) progressDetailsLabel.TextStyle = fyne.TextStyle{ Bold: false, Italic: true, Monospace: false, } progressDetailsLabelCentered := getWidgetCentered(progressDetailsLabel) estimatedTimeRemainingLabel := widget.NewLabelWithData(estimatedTimeRemainingBinding) estimatedTimeRemainingLabel.TextStyle = fyne.TextStyle{ Bold: false, Italic: true, Monospace: false, } estimatedTimeRemainingLabelCentered := getWidgetCentered(estimatedTimeRemainingLabel) // We set this bool to true to stop the trainModel process var trainModelIsStoppedBoolMutex sync.RWMutex trainModelIsStoppedBool := false cancelButton := getWidgetCentered(widget.NewButtonWithIcon("Cancel", theme.CancelIcon(), func(){ trainModelIsStoppedBoolMutex.Lock() trainModelIsStoppedBool = true trainModelIsStoppedBoolMutex.Unlock() previousPage() })) page := container.NewVBox(title, widget.NewSeparator(), loadingBar, progressDetailsTitle, progressDetailsLabelCentered, estimatedTimeRemainingLabelCentered, widget.NewSeparator(), cancelButton) window.SetContent(page) trainModelFunction := func(){ var processProgressMutex sync.RWMutex // This stores the amount of progress which has been completed (0-1) processProgress := float64(0) startUpdateTimeRemainingDisplayFunction := func(){ // This function updates the estimated time remaining label binding updateTimeRemainingDisplayFunction := func()error{ startTime := time.Now().Unix() for{ trainModelIsStoppedBoolMutex.RLock() trainModelIsStopped := trainModelIsStoppedBool trainModelIsStoppedBoolMutex.RUnlock() if (trainModelIsStopped == true){ // User exited the process/Process has completed return nil } processProgressMutex.RLock() currentProcessProgress := processProgress processProgressMutex.RUnlock() if (currentProcessProgress == 0){ estimatedTimeRemainingBinding.Set("Calculating required time...") time.Sleep(time.Second) continue } // We calculate how long we think it will take for the process to complete currentTime := time.Now().Unix() secondsElapsed := currentTime - startTime // processProgress is a float64 which stores the progress as a value between 0-1 // To get the estimated total time the process will take, we divide the seconds elapsed by the proportion of progress // For example: // 0.1 (10%) at 10 seconds == Total process will take 100 seconds // 0.5 (50%) at 20 seconds == Total process will take 40 seconds totalSeconds := float64(secondsElapsed)/currentProcessProgress estimatedSecondsRemaining := int64(totalSeconds) - secondsElapsed estimatedTimeRemainingTranslated, err := helpers.ConvertUnixTimeDurationToUnitsTimeTranslated(estimatedSecondsRemaining, false) if (err != nil) { return err } estimatedTimeRemainingBinding.Set("Estimated Time Remaining: " + estimatedTimeRemainingTranslated) time.Sleep(time.Second) } // This should never be reached return errors.New("updateTimeRemainingDisplayFunction loop has broken.") } err := updateTimeRemainingDisplayFunction() if (err != nil){ trainModelIsStoppedBoolMutex.Lock() trainModelIsStoppedBool = true trainModelIsStoppedBoolMutex.Unlock() setErrorEncounteredPage(window, err, previousPage) return } } go startUpdateTimeRemainingDisplayFunction() //Outputs: // -bool: Process completed (true == was not stopped mid-way) // -error trainModel := func()(bool, error){ _, err := localFilesystem.CreateFolder("./TrainedModels") if (err != nil) { return false, err } trainingSetFilepathsList, _, err := getTrainingAndTestingDataFilepathLists(traitName) if (err != nil) { return false, err } // Now we deterministically randomize the order of the trainingSetFilepathsList pseudorandomNumberGenerator := mathRand.New(mathRand.NewPCG(1, 2)) pseudorandomNumberGenerator.Shuffle(len(trainingSetFilepathsList), func(i int, j int){ trainingSetFilepathsList[i], trainingSetFilepathsList[j] = trainingSetFilepathsList[j], trainingSetFilepathsList[i] }) // We create a new neural network object to train neuralNetworkObject, err := geneticPrediction.GetNewUntrainedNeuralNetworkObject(traitName) if (err != nil) { return false, err } numberOfTrainingDatas := len(trainingSetFilepathsList) numberOfTrainingDatasString := helpers.ConvertIntToString(numberOfTrainingDatas) // This keeps track of how far along we are in training trainingDataIndex := 0 // Outputs: // -bool: User stopped training // -bool: Another training data exists // -geneticPrediction.TrainingData // -error getNextTrainingDataFunction := func()(bool, bool, geneticPrediction.TrainingData, error){ trainModelIsStoppedBoolMutex.RLock() trainModelIsStopped := trainModelIsStoppedBool trainModelIsStoppedBoolMutex.RUnlock() if (trainModelIsStopped == true){ // User exited the process return true, false, geneticPrediction.TrainingData{}, nil } if (trainingDataIndex == numberOfTrainingDatas){ // We are done training. return false, false, geneticPrediction.TrainingData{}, nil } trainingDataFilepath := trainingSetFilepathsList[trainingDataIndex] fileExists, fileContents, err := localFilesystem.GetFileContents(trainingDataFilepath) if (err != nil) { return false, false, geneticPrediction.TrainingData{}, err } if (fileExists == false){ return false, false, geneticPrediction.TrainingData{}, errors.New("TrainingData file not found: " + trainingDataFilepath) } trainingDataObject, err := geneticPrediction.DecodeBytesToTrainingDataObject(fileContents) if (err != nil) { return false, false, geneticPrediction.TrainingData{}, err } trainingDataIndex += 1 numberOfExamplesTrainedString := helpers.ConvertIntToString(trainingDataIndex + 1) numberOfExamplesProgress := "Trained " + numberOfExamplesTrainedString + "/" + numberOfTrainingDatasString + " Examples" progressDetailsBinding.Set(numberOfExamplesProgress) newProgressFloat64 := float64(trainingDataIndex)/float64(numberOfTrainingDatas) err = progressPercentageBinding.Set(newProgressFloat64) if (err != nil) { return false, false, geneticPrediction.TrainingData{}, err } processProgressMutex.Lock() processProgress = newProgressFloat64 processProgressMutex.Unlock() return false, true, trainingDataObject, nil } traitObject, err := traits.GetTraitObject(traitName) if (err != nil) { return false, err } getTraitIsNumericBool := func()bool{ traitIsDiscreteOrNumeric := traitObject.DiscreteOrNumeric if (traitIsDiscreteOrNumeric == "Numeric"){ return true } return false } traitIsNumeric := getTraitIsNumericBool() processCompleted, err := geneticPrediction.TrainNeuralNetwork(traitName, traitIsNumeric, neuralNetworkObject, getNextTrainingDataFunction) if (err != nil) { return false, err } if (processCompleted == false){ return false, nil } // Network training is complete. // We now save the neural network as a .gob file neuralNetworkBytes, err := geneticPrediction.EncodeNeuralNetworkObjectToBytes(*neuralNetworkObject) if (err != nil) { return false, err } traitNameWithoutWhitespaces := strings.ReplaceAll(traitName, " ", "") neuralNetworkFilename := traitNameWithoutWhitespaces + "Model.gob" err = localFilesystem.CreateOrOverwriteFile(neuralNetworkBytes, "./TrainedModels/", neuralNetworkFilename) if (err != nil) { return false, err } progressPercentageBinding.Set(1) return true, nil } processIsComplete, err := trainModel() if (err != nil){ trainModelIsStoppedBoolMutex.Lock() trainModelIsStoppedBool = true trainModelIsStoppedBoolMutex.Unlock() setErrorEncounteredPage(window, err, previousPage) return } if (processIsComplete == false){ // User exited the page return } setTrainModelIsCompletePage(window) } go trainModelFunction() } func setTrainModelIsCompletePage(window fyne.Window){ title := getBoldLabelCentered("Training Model Is Complete") description1 := getLabelCentered("Model training is complete!") description2 := getLabelCentered("The model has been saved in the TrainedModels folder.") exitButton := getWidgetCentered(widget.NewButtonWithIcon("Exit", theme.CancelIcon(), func(){ setHomePage(window) })) page := container.NewVBox(title, widget.NewSeparator(), description1, description2, exitButton) window.SetContent(page) } func setTestModelsPage(window fyne.Window, previousPage func()){ currentPage := func(){setTestModelsPage(window, previousPage)} title := getBoldLabelCentered("Test Models") backButton := getBackButtonCentered(previousPage) description1 := getLabelCentered("Press the button below to begin testing a genetic model.") description2 := getLabelCentered("This will test each neural network using user training data examples.") description3 := getLabelCentered("The testing data is not used to train the models.") description4 := getLabelCentered("The results of the testing will be displayed at the end.") description5 := getLabelCentered("The results will also be saved in the ModelAccuracies folder.") description6 := getLabelCentered("You must select a trait model to test.") traitNamesList := []string{"Eye Color", "Lactose Tolerance", "Height"} traitNameSelector := widget.NewSelect(traitNamesList, nil) beginTestingButton := getWidgetCentered(widget.NewButtonWithIcon("Begin Testing Model", theme.MediaPlayIcon(), func(){ selectedTraitIndex := traitNameSelector.SelectedIndex() if (selectedTraitIndex < 0){ title := "No Trait Selected" dialogMessage1 := getLabelCentered("You must select a trait model to test.") dialogContent := container.NewVBox(dialogMessage1) dialog.ShowCustom(title, "Close", dialogContent, window) return } traitName := traitNameSelector.Selected setStartAndMonitorTestModelPage(window, traitName, currentPage) })) traitNameSelectorCentered := getWidgetCentered(traitNameSelector) page := container.NewVBox(title, backButton, widget.NewSeparator(), description1, description2, description3, description4, description5, description6, widget.NewSeparator(), traitNameSelectorCentered, widget.NewSeparator(), beginTestingButton) window.SetContent(page) } func setStartAndMonitorTestModelPage(window fyne.Window, traitName string, previousPage func()){ title := getBoldLabelCentered("Testing Model") progressDetailsBinding := binding.NewString() progressPercentageBinding := binding.NewFloat() loadingBar := getWidgetCentered(widget.NewProgressBarWithData(progressPercentageBinding)) progressDetailsTitle := getBoldLabelCentered("Progress Details:") progressDetailsLabel := widget.NewLabelWithData(progressDetailsBinding) progressDetailsLabel.TextStyle = fyne.TextStyle{ Bold: false, Italic: true, Monospace: false, } progressDetailsLabelCentered := getWidgetCentered(progressDetailsLabel) // We set this bool to true to stop the testModel process var testModelIsStoppedBoolMutex sync.RWMutex testModelIsStoppedBool := false cancelButton := getWidgetCentered(widget.NewButtonWithIcon("Cancel", theme.CancelIcon(), func(){ testModelIsStoppedBoolMutex.Lock() testModelIsStoppedBool = true testModelIsStoppedBoolMutex.Unlock() previousPage() })) page := container.NewVBox(title, widget.NewSeparator(), loadingBar, progressDetailsTitle, progressDetailsLabelCentered, widget.NewSeparator(), cancelButton) window.SetContent(page) testModelFunction := func(){ traitObject, err := traits.GetTraitObject(traitName) if (err != nil){ setErrorEncounteredPage(window, err, previousPage) return } traitIsDiscreteOrNumeric := traitObject.DiscreteOrNumeric if (traitIsDiscreteOrNumeric == "Discrete"){ //Outputs: // -bool: Process completed (true == was not stopped mid-way) // -geneticPrediction.DiscreteTraitPredictionAccuracyInfoMap // -error testModel := func()(bool, geneticPrediction.DiscreteTraitPredictionAccuracyInfoMap, error){ type TraitAccuracyStatisticsValue struct{ // This stores the quantity of examples of this outcome QuantityOfExamples int // This stores the quantity of predictions that were made for this outcome // In other words: The quantity of instances where our model predicted this outcome QuantityOfPredictions int // This stores the quantity of predictions that were correct when the genome had this outcome QuantityOfCorrectGenomePredictions int // This stores the quantity of predictions that were correct when the model predicted this outcome QuantityOfCorrectOutcomePredictions int } // We use this map to count up the information about predictions // We use information from this map to construct the final accuracy information map traitPredictionInfoMap := make(map[geneticPrediction.DiscreteTraitOutcomeInfo]TraitAccuracyStatisticsValue) _, testingSetFilepathsList, err := getTrainingAndTestingDataFilepathLists(traitName) if (err != nil) { return false, nil, err } traitNameWithoutWhitespaces := strings.ReplaceAll(traitName, " ", "") // We read the trained model for this trait modelFilename := traitNameWithoutWhitespaces + "Model.gob" trainedModelFilepath := goFilepath.Join("./TrainedModels/", modelFilename) fileExists, fileContents, err := localFilesystem.GetFileContents(trainedModelFilepath) if (err != nil) { return false, nil, err } if (fileExists == false){ return false, nil, errors.New("TrainedModel not found: " + trainedModelFilepath) } neuralNetworkObject, err := geneticPrediction.DecodeBytesToNeuralNetworkObject(fileContents) if (err != nil) { return false, nil, err } numberOfTrainingDatas := len(testingSetFilepathsList) numberOfTrainingDatasString := helpers.ConvertIntToString(numberOfTrainingDatas) finalIndex := numberOfTrainingDatas - 1 for index, filePath := range testingSetFilepathsList{ testModelIsStoppedBoolMutex.RLock() testModelIsStopped := testModelIsStoppedBool testModelIsStoppedBoolMutex.RUnlock() if (testModelIsStopped == true){ // User exited the process return false, nil, nil } fileExists, fileContents, err := localFilesystem.GetFileContents(filePath) if (err != nil) { return false, nil, err } if (fileExists == false){ return false, nil, errors.New("TrainingData file not found: " + filePath) } trainingDataObject, err := geneticPrediction.DecodeBytesToTrainingDataObject(fileContents) if (err != nil) { return false, nil, err } trainingDataInputLayer := trainingDataObject.InputLayer trainingDataExpectedOutputLayer := trainingDataObject.OutputLayer predictionLayer, err := geneticPrediction.GetNeuralNetworkRawPrediction(&neuralNetworkObject, false, trainingDataInputLayer) if (err != nil) { return false, nil, err } numberOfPredictionNeurons := len(predictionLayer) if (len(trainingDataExpectedOutputLayer) != numberOfPredictionNeurons){ return false, nil, errors.New("Neural network prediction output length does not match expected output length.") } correctOutcomeName, err := geneticPrediction.GetDiscreteOutcomeNameFromOutputLayer(traitName, true, trainingDataExpectedOutputLayer) if (err != nil) { return false, nil, err } predictedOutcomeName, err := geneticPrediction.GetDiscreteOutcomeNameFromOutputLayer(traitName, true, predictionLayer) if (err != nil) { return false, nil, err } getPredictionIsCorrectBool := func()bool{ if (predictedOutcomeName == correctOutcomeName){ return true } return false } predictionIsCorrect := getPredictionIsCorrectBool() numberOfKnownLoci, numberOfKnownAndPhasedLoci, numberOfLoci, err := geneticPrediction.GetLociInfoFromNetworkInputLayer(trainingDataInputLayer) if (err != nil) { return false, nil, err } proportionOfLociTested := float64(numberOfKnownLoci)/float64(numberOfLoci) percentageOfLociTested := int(100*proportionOfLociTested) proportionOfPhasedLoci := float64(numberOfKnownAndPhasedLoci)/float64(numberOfKnownLoci) percentageOfPhasedLoci := int(100*proportionOfPhasedLoci) { // We first add the information to the map for the correct outcome newTraitOutcomeInfo_CorrectOutcome := geneticPrediction.DiscreteTraitOutcomeInfo{ OutcomeName: correctOutcomeName, PercentageOfLociTested: percentageOfLociTested, PercentageOfPhasedLoci: percentageOfPhasedLoci, } getTraitAccuracyStatisticsValue_CorrectOutcome := func()TraitAccuracyStatisticsValue{ existingTraitAccuracyStatisticsValue, exists := traitPredictionInfoMap[newTraitOutcomeInfo_CorrectOutcome] if (exists == false){ newTraitAccuracyStatisticsValue := TraitAccuracyStatisticsValue{} return newTraitAccuracyStatisticsValue } return existingTraitAccuracyStatisticsValue } traitAccuracyStatisticsValue := getTraitAccuracyStatisticsValue_CorrectOutcome() traitAccuracyStatisticsValue.QuantityOfExamples += 1 if (predictionIsCorrect == true){ traitAccuracyStatisticsValue.QuantityOfCorrectGenomePredictions += 1 } traitPredictionInfoMap[newTraitOutcomeInfo_CorrectOutcome] = traitAccuracyStatisticsValue } { // We now add the information to the map for the predicted outcome newTraitOutcomeInfo_PredictedOutcome := geneticPrediction.DiscreteTraitOutcomeInfo{ OutcomeName: predictedOutcomeName, PercentageOfLociTested: percentageOfLociTested, PercentageOfPhasedLoci: percentageOfPhasedLoci, } getTraitAccuracyStatisticsValue_PredictedOutcome := func()TraitAccuracyStatisticsValue{ existingTraitAccuracyStatisticsValue, exists := traitPredictionInfoMap[newTraitOutcomeInfo_PredictedOutcome] if (exists == false){ newTraitAccuracyStatisticsValue := TraitAccuracyStatisticsValue{} return newTraitAccuracyStatisticsValue } return existingTraitAccuracyStatisticsValue } traitAccuracyStatisticsValue := getTraitAccuracyStatisticsValue_PredictedOutcome() traitAccuracyStatisticsValue.QuantityOfPredictions += 1 if (predictionIsCorrect == true){ traitAccuracyStatisticsValue.QuantityOfCorrectOutcomePredictions += 1 } traitPredictionInfoMap[newTraitOutcomeInfo_PredictedOutcome] = traitAccuracyStatisticsValue } exampleIndexString := helpers.ConvertIntToString(index+1) numberOfExamplesProgress := "Tested " + exampleIndexString + "/" + numberOfTrainingDatasString + " Examples" progressDetailsBinding.Set(numberOfExamplesProgress) newProgressFloat64 := float64(index)/float64(finalIndex) progressPercentageBinding.Set(newProgressFloat64) } // Now we construct the TraitAccuracyInfoMap // This map stores the accuracy for each outcome traitPredictionAccuracyInfoMap := make(map[geneticPrediction.DiscreteTraitOutcomeInfo]geneticPrediction.DiscreteTraitPredictionAccuracyInfo) for traitPredictionInfo, value := range traitPredictionInfoMap{ quantityOfExamples := value.QuantityOfExamples quantityOfPredictions := value.QuantityOfPredictions quantityOfCorrectGenomePredictions := value.QuantityOfCorrectGenomePredictions quantityOfCorrectOutcomePredictions := value.QuantityOfCorrectOutcomePredictions if (quantityOfCorrectGenomePredictions > quantityOfExamples){ return false, nil, errors.New("traitPredictionInfoMap contains quantityOfCorrectGenomePredictions > quantityOfExamples") } if (quantityOfCorrectOutcomePredictions > quantityOfPredictions){ return false, nil, errors.New("traitPredictionInfoMap contains quantityOfCorrectOutcomePredictions > quantityOfPredictions") } newTraitPredictionAccuracyInfo := geneticPrediction.DiscreteTraitPredictionAccuracyInfo{ QuantityOfExamples: quantityOfExamples, QuantityOfPredictions: quantityOfPredictions, } if (quantityOfExamples > 0){ proportionOfCorrectGenomePredictions := float64(quantityOfCorrectGenomePredictions)/float64(quantityOfExamples) percentageOfCorrectGenomePredictions := int(100*proportionOfCorrectGenomePredictions) newTraitPredictionAccuracyInfo.ProbabilityOfCorrectGenomePrediction = percentageOfCorrectGenomePredictions } if (quantityOfPredictions > 0){ proportionOfCorrectOutcomePredictions := float64(quantityOfCorrectOutcomePredictions)/float64(quantityOfPredictions) percentageOfCorrectOutcomePredictions := int(100*proportionOfCorrectOutcomePredictions) newTraitPredictionAccuracyInfo.ProbabilityOfCorrectOutcomePrediction = percentageOfCorrectOutcomePredictions } traitPredictionAccuracyInfoMap[traitPredictionInfo] = newTraitPredictionAccuracyInfo } // Testing is complete. // We save the info map as a file in the ModelAccuracies folder fileBytes, err := geneticPrediction.EncodeDiscreteTraitPredictionAccuracyInfoMapToBytes(traitPredictionAccuracyInfoMap) if (err != nil) { return false, nil, err } _, err = localFilesystem.CreateFolder("./ModelAccuracies") if (err != nil) { return false, nil, err } modelAccuracyFilename := traitNameWithoutWhitespaces + "ModelAccuracy.gob" err = localFilesystem.CreateOrOverwriteFile(fileBytes, "./ModelAccuracies/", modelAccuracyFilename) if (err != nil) { return false, nil, err } progressPercentageBinding.Set(1) return true, traitPredictionAccuracyInfoMap, nil } processIsComplete, traitPredictionAccuracyInfoMap, err := testModel() if (err != nil){ setErrorEncounteredPage(window, err, previousPage) return } if (processIsComplete == false){ // User exited the page return } setViewModelTestingDiscreteTraitResultsPage(window, traitName, traitPredictionAccuracyInfoMap, previousPage) } else { // traitIsDiscreteOrNumeric == "Numeric" //Outputs: // -bool: Process completed (true == was not stopped mid-way) // -geneticPrediction.NumericTraitPredictionAccuracyInfoMap // -error testModel := func()(bool, geneticPrediction.NumericTraitPredictionAccuracyInfoMap, error){ // We use this map to count up the information about predictions // We use information from this map to construct the final accuracy information map // Map Structure: NumericTraitOutcomeInfo -> List of true outcomes traitPredictionInfoMap := make(map[geneticPrediction.NumericTraitOutcomeInfo][]float64) _, testingSetFilepathsList, err := getTrainingAndTestingDataFilepathLists(traitName) if (err != nil) { return false, nil, err } traitNameWithoutWhitespaces := strings.ReplaceAll(traitName, " ", "") // We read the trained model for this trait modelFilename := traitNameWithoutWhitespaces + "Model.gob" trainedModelFilepath := goFilepath.Join("./TrainedModels/", modelFilename) fileExists, fileContents, err := localFilesystem.GetFileContents(trainedModelFilepath) if (err != nil) { return false, nil, err } if (fileExists == false){ return false, nil, errors.New("TrainedModel not found: " + trainedModelFilepath) } neuralNetworkObject, err := geneticPrediction.DecodeBytesToNeuralNetworkObject(fileContents) if (err != nil) { return false, nil, err } numberOfTrainingDatas := len(testingSetFilepathsList) numberOfTrainingDatasString := helpers.ConvertIntToString(numberOfTrainingDatas) finalIndex := numberOfTrainingDatas - 1 for index, filePath := range testingSetFilepathsList{ testModelIsStoppedBoolMutex.RLock() testModelIsStopped := testModelIsStoppedBool testModelIsStoppedBoolMutex.RUnlock() if (testModelIsStopped == true){ // User exited the process return false, nil, nil } fileExists, fileContents, err := localFilesystem.GetFileContents(filePath) if (err != nil) { return false, nil, err } if (fileExists == false){ return false, nil, errors.New("TrainingData file not found: " + filePath) } trainingDataObject, err := geneticPrediction.DecodeBytesToTrainingDataObject(fileContents) if (err != nil) { return false, nil, err } trainingDataInputLayer := trainingDataObject.InputLayer trainingDataExpectedOutputLayer := trainingDataObject.OutputLayer predictionLayer, err := geneticPrediction.GetNeuralNetworkRawPrediction(&neuralNetworkObject, false, trainingDataInputLayer) if (err != nil) { return false, nil, err } if (len(predictionLayer) != 1){ return false, nil, errors.New("Neural network numeric prediction output layer length is not 1.") } if (len(trainingDataExpectedOutputLayer) != 1){ return false, nil, errors.New("Neural network training data prediction output layer length is not 1.") } correctOutcomeValue, err := geneticPrediction.GetNumericOutcomeValueFromOutputLayer(traitName, trainingDataExpectedOutputLayer) if (err != nil) { return false, nil, err } predictedOutcomeValue, err := geneticPrediction.GetNumericOutcomeValueFromOutputLayer(traitName, predictionLayer) if (err != nil) { return false, nil, err } numberOfKnownLoci, numberOfKnownAndPhasedLoci, numberOfLoci, err := geneticPrediction.GetLociInfoFromNetworkInputLayer(trainingDataInputLayer) if (err != nil) { return false, nil, err } proportionOfLociTested := float64(numberOfKnownLoci)/float64(numberOfLoci) percentageOfLociTested := int(100*proportionOfLociTested) proportionOfPhasedLoci := float64(numberOfKnownAndPhasedLoci)/float64(numberOfKnownLoci) percentageOfPhasedLoci := int(100*proportionOfPhasedLoci) newNumericTraitOutcomeInfo := geneticPrediction.NumericTraitOutcomeInfo{ OutcomeValue: predictedOutcomeValue, PercentageOfLociTested: percentageOfLociTested, PercentageOfPhasedLoci: percentageOfPhasedLoci, } existingList, exists := traitPredictionInfoMap[newNumericTraitOutcomeInfo] if (exists == false){ traitPredictionInfoMap[newNumericTraitOutcomeInfo] = []float64{correctOutcomeValue} } else { existingList = append(existingList, correctOutcomeValue) traitPredictionInfoMap[newNumericTraitOutcomeInfo] = existingList } exampleIndexString := helpers.ConvertIntToString(index+1) numberOfExamplesProgress := "Tested " + exampleIndexString + "/" + numberOfTrainingDatasString + " Examples" progressDetailsBinding.Set(numberOfExamplesProgress) newProgressFloat64 := float64(index)/float64(finalIndex) progressPercentageBinding.Set(newProgressFloat64) } // Now we construct the TraitAccuracyInfoMap // This map stores the accuracy for each outcome traitPredictionAccuracyInfoMap := make(map[geneticPrediction.NumericTraitOutcomeInfo]geneticPrediction.NumericTraitPredictionAccuracyRangesMap) for traitPredictionInfo, realOutcomesList := range traitPredictionInfoMap{ if (len(realOutcomesList) == 0){ return false, nil, errors.New("traitPredictionInfoMap contains empty realOutcomesList.") } // This is the predicted height value for this set of real outcomes predictionValue := traitPredictionInfo.OutcomeValue // Map Structure: Accuracy Percentage (AP) -> Amount needed to deviate from prediction // for the value to be accurate (AP)% of the time newNumericTraitPredictionAccuracyRangesMap := make(map[int]float64) rangeDistance := float64(0) for { rangeMin := predictionValue - rangeDistance rangeMax := predictionValue + rangeDistance valuesInRangeList := make([]float64, 0) valuesOutOfRangeList := make([]float64, 0) for _, outcomeValue := range realOutcomesList{ if (outcomeValue <= rangeMax && outcomeValue >= rangeMin){ valuesInRangeList = append(valuesInRangeList, outcomeValue) } else { valuesOutOfRangeList = append(valuesOutOfRangeList, outcomeValue) } } quantityOfValuesInRange := len(valuesInRangeList) totalQuantityOfValues := len(realOutcomesList) proportionOfValuesInRange := float64(quantityOfValuesInRange)/float64(totalQuantityOfValues) percentageOfValuesInRange := proportionOfValuesInRange * 100 if (percentageOfValuesInRange >= 1){ percentageOfValuesInRangeInt := int(percentageOfValuesInRange) newNumericTraitPredictionAccuracyRangesMap[percentageOfValuesInRangeInt] = rangeDistance } if (quantityOfValuesInRange == totalQuantityOfValues){ newNumericTraitPredictionAccuracyRangesMap[100] = rangeDistance break } // Now we increase rangeDistance // We find the distance to the next closest item in the list that isn't already in our range nearestValueDistance := float64(0) for index, outcomeValue := range valuesOutOfRangeList{ distance := math.Abs(predictionValue - outcomeValue) if (index == 0 || distance < nearestValueDistance){ nearestValueDistance = distance } } rangeDistance += nearestValueDistance } traitPredictionAccuracyInfoMap[traitPredictionInfo] = newNumericTraitPredictionAccuracyRangesMap } // Testing is complete. // We save the info map as a file in the ModelAccuracies folder fileBytes, err := geneticPrediction.EncodeNumericTraitPredictionAccuracyInfoMapToBytes(traitPredictionAccuracyInfoMap) if (err != nil) { return false, nil, err } _, err = localFilesystem.CreateFolder("./ModelAccuracies") if (err != nil) { return false, nil, err } modelAccuracyFilename := traitNameWithoutWhitespaces + "ModelAccuracy.gob" err = localFilesystem.CreateOrOverwriteFile(fileBytes, "./ModelAccuracies/", modelAccuracyFilename) if (err != nil) { return false, nil, err } progressPercentageBinding.Set(1) return true, traitPredictionAccuracyInfoMap, nil } processIsComplete, traitPredictionAccuracyInfoMap, err := testModel() if (err != nil){ setErrorEncounteredPage(window, err, previousPage) return } if (processIsComplete == false){ // User exited the page return } setViewModelTestingNumericTraitResultsPage(window, traitName, traitPredictionAccuracyInfoMap, previousPage) } } go testModelFunction() } // This is a page to view the details of testing for a specific trait's model func setViewModelTestingDiscreteTraitResultsPage(window fyne.Window, traitName string, traitAccuracyInfoMap geneticPrediction.DiscreteTraitPredictionAccuracyInfoMap, exitPage func()){ title := getBoldLabelCentered("Discrete Trait Prediction Accuracy Details") exitButton := getWidgetCentered(widget.NewButtonWithIcon("Exit", theme.CancelIcon(), exitPage)) description1 := getLabelCentered("The results of the prediction accuracy for this trait are below.") traitNameTitle := widget.NewLabel("Trait Name:") traitNameLabel := getBoldLabel(traitName) traitNameRow := container.NewHBox(layout.NewSpacer(), traitNameTitle, traitNameLabel, layout.NewSpacer()) description2 := getLabelCentered("Prediction accuracy values are a pair of Genome Accuracy/Outcome Accuracy.") description3 := getLabelCentered("Genome Accuracy is the probability that the model will predict a genome's trait value correctly.") description4 := getLabelCentered("Outcome Accuracy is the probability that a trait prediction that the model makes is correct.") getResultsGrid := func()(*fyne.Container, error){ outcomeNameTitle := getItalicLabelCentered("Outcome Name") emptyLabel1 := widget.NewLabel("") predictionAccuracyTitle1 := getItalicLabelCentered("Prediction Accuracy") knownLociLabel_0to33 := getItalicLabelCentered("0-33% Known Loci") predictionAccuracyTitle2 := getItalicLabelCentered("Prediction Accuracy") knownLociLabel_34to66 := getItalicLabelCentered("34-66% Known Loci") predictionAccuracyTitle3 := getItalicLabelCentered("Prediction Accuracy") knownLociLabel_67to100 := getItalicLabelCentered("67-100% Known Loci") outcomeNameColumn := container.NewVBox(outcomeNameTitle, emptyLabel1, widget.NewSeparator()) predictionAccuracyColumn_0to33 := container.NewVBox(predictionAccuracyTitle1, knownLociLabel_0to33, widget.NewSeparator()) predictionAccuracyColumn_34to66 := container.NewVBox(predictionAccuracyTitle2, knownLociLabel_34to66, widget.NewSeparator()) predictionAccuracyColumn_67to100 := container.NewVBox(predictionAccuracyTitle3, knownLociLabel_67to100, widget.NewSeparator()) traitObject, err := traits.GetTraitObject(traitName) if (err != nil) { return nil, err } traitIsDiscreteOrNumeric := traitObject.DiscreteOrNumeric if (traitIsDiscreteOrNumeric != "Discrete"){ return nil, errors.New("setViewModelTestingDiscreteTraitResultsPage called with non-discrete trait: " + traitName) } outcomeNamesList := traitObject.OutcomesList for _, outcomeName := range outcomeNamesList{ outcomeNameLabel := getBoldLabelCentered(outcomeName) // We use the below variables to sum up the accuracy percentages so we can average them genomePredictionAccuracySum_0to33 := 0 genomeExampleCount_0to33 := 0 outcomePredictionAccuracySum_0to33 := 0 outcomePredictionCount_0to33 := 0 genomePredictionAccuracySum_34to66 := 0 genomeExampleCount_34to66 := 0 outcomePredictionAccuracySum_34to66 := 0 outcomePredictionCount_34to66 := 0 genomePredictionAccuracySum_67to100 := 0 genomeExampleCount_67to100 := 0 outcomePredictionAccuracySum_67to100 := 0 outcomePredictionCount_67to100 := 0 for traitOutcomeInfo, traitPredictionAccuracyInfo := range traitAccuracyInfoMap{ currentOutcomeName := traitOutcomeInfo.OutcomeName if (currentOutcomeName != outcomeName){ continue } percentageOfLociTested := traitOutcomeInfo.PercentageOfLociTested quantityOfExamples := traitPredictionAccuracyInfo.QuantityOfExamples quantityOfPredictions := traitPredictionAccuracyInfo.QuantityOfPredictions genomePredictionAccuracyPercentage := traitPredictionAccuracyInfo.ProbabilityOfCorrectGenomePrediction outcomePredictionAccuracyPercentage := traitPredictionAccuracyInfo.ProbabilityOfCorrectOutcomePrediction if (percentageOfLociTested <= 33){ genomePredictionAccuracySum_0to33 += (genomePredictionAccuracyPercentage * quantityOfExamples) genomeExampleCount_0to33 += quantityOfExamples outcomePredictionAccuracySum_0to33 += (outcomePredictionAccuracyPercentage * quantityOfPredictions) outcomePredictionCount_0to33 += quantityOfPredictions } else if (percentageOfLociTested > 33 && percentageOfLociTested <= 66){ genomePredictionAccuracySum_34to66 += (genomePredictionAccuracyPercentage * quantityOfExamples) genomeExampleCount_34to66 += quantityOfExamples outcomePredictionAccuracySum_34to66 += (outcomePredictionAccuracyPercentage * quantityOfPredictions) outcomePredictionCount_34to66 += quantityOfPredictions } else { genomePredictionAccuracySum_67to100 += (genomePredictionAccuracyPercentage * quantityOfExamples) genomeExampleCount_67to100 += quantityOfExamples outcomePredictionAccuracySum_67to100 += (outcomePredictionAccuracyPercentage * quantityOfPredictions) outcomePredictionCount_67to100 += quantityOfPredictions } } getAverageAccuracyText := func(accuracySum int, predictionCount int)string{ if (predictionCount == 0){ return "Unknown" } averageAccuracy := accuracySum/predictionCount averageAccuracyString := helpers.ConvertIntToString(averageAccuracy) result := averageAccuracyString + "%" return result } genomeAverageAccuracyText_0to33 := getAverageAccuracyText(genomePredictionAccuracySum_0to33, genomeExampleCount_0to33) genomeAverageAccuracyText_34to66 := getAverageAccuracyText(genomePredictionAccuracySum_34to66, genomeExampleCount_34to66) genomeAverageAccuracyText_67to100 := getAverageAccuracyText(genomePredictionAccuracySum_67to100, genomeExampleCount_67to100) outcomeAverageAccuracyText_0to33 := getAverageAccuracyText(outcomePredictionAccuracySum_0to33, outcomePredictionCount_0to33) outcomeAverageAccuracyText_34to66 := getAverageAccuracyText(outcomePredictionAccuracySum_34to66, outcomePredictionCount_34to66) outcomeAverageAccuracyText_67to100 := getAverageAccuracyText(outcomePredictionAccuracySum_67to100, outcomePredictionCount_67to100) averageAccuracyLabel_0to33 := getBoldLabelCentered(genomeAverageAccuracyText_0to33 + "/" + outcomeAverageAccuracyText_0to33) averageAccuracyLabel_34to66 := getBoldLabelCentered(genomeAverageAccuracyText_34to66 + "/" + outcomeAverageAccuracyText_34to66) averageAccuracyLabel_67to100 := getBoldLabelCentered(genomeAverageAccuracyText_67to100 + "/" + outcomeAverageAccuracyText_67to100) outcomeNameColumn.Add(outcomeNameLabel) predictionAccuracyColumn_0to33.Add(averageAccuracyLabel_0to33) predictionAccuracyColumn_34to66.Add(averageAccuracyLabel_34to66) predictionAccuracyColumn_67to100.Add(averageAccuracyLabel_67to100) outcomeNameColumn.Add(widget.NewSeparator()) predictionAccuracyColumn_0to33.Add(widget.NewSeparator()) predictionAccuracyColumn_34to66.Add(widget.NewSeparator()) predictionAccuracyColumn_67to100.Add(widget.NewSeparator()) } resultsGrid := container.NewHBox(layout.NewSpacer(), outcomeNameColumn, predictionAccuracyColumn_0to33, predictionAccuracyColumn_34to66, predictionAccuracyColumn_67to100, layout.NewSpacer()) return resultsGrid, nil } resultsGrid, err := getResultsGrid() if (err != nil){ setErrorEncounteredPage(window, err, func(){setHomePage(window)}) return } page := container.NewVBox(title, exitButton, widget.NewSeparator(), description1, widget.NewSeparator(), traitNameRow, widget.NewSeparator(), description2, description3, description4, widget.NewSeparator(), resultsGrid) window.SetContent(page) } // This is a page to view the details of testing for a specific trait's model func setViewModelTestingNumericTraitResultsPage(window fyne.Window, traitName string, traitAccuracyInfoMap geneticPrediction.NumericTraitPredictionAccuracyInfoMap, exitPage func()){ title := getBoldLabelCentered("Numeric Trait Prediction Accuracy Details") exitButton := getWidgetCentered(widget.NewButtonWithIcon("Exit", theme.CancelIcon(), exitPage)) description1 := getLabelCentered("The results of the prediction accuracy for this trait are below.") traitNameTitle := widget.NewLabel("Trait Name:") traitNameLabel := getBoldLabel(traitName) traitNameRow := container.NewHBox(layout.NewSpacer(), traitNameTitle, traitNameLabel, layout.NewSpacer()) description2 := getLabelCentered("Each value is a range that the prediction must be widened by to be accurate X% of the time.") description3 := getLabelCentered("For example, for a height prediction to be accurate 90% of the time, allow a +/-10 cm range.") getResultsGrid := func()(*fyne.Container, error){ probabilityOfTitle := getItalicLabelCentered("Probability Of") correctPredictionTitle := getItalicLabelCentered("Correct Prediction") accuracyRangeTitle1 := getItalicLabelCentered("Accuracy Range") knownLociLabel_0to33 := getItalicLabelCentered("0-33% Known Loci") accuracyRangeTitle2 := getItalicLabelCentered("Accuracy Range") knownLociLabel_34to66 := getItalicLabelCentered("34-66% Known Loci") accuracyRangeTitle3 := getItalicLabelCentered("Accuracy Range") knownLociLabel_67to100 := getItalicLabelCentered("67-100% Known Loci") probabilityOfCorrectPredictionColumn := container.NewVBox(probabilityOfTitle, correctPredictionTitle, widget.NewSeparator()) accuracyRangeColumn_0to33 := container.NewVBox(accuracyRangeTitle1, knownLociLabel_0to33, widget.NewSeparator()) accuracyRangeColumn_34to66 := container.NewVBox(accuracyRangeTitle2, knownLociLabel_34to66, widget.NewSeparator()) accuracyRangeColumn_67to100 := container.NewVBox(accuracyRangeTitle3, knownLociLabel_67to100, widget.NewSeparator()) traitObject, err := traits.GetTraitObject(traitName) if (err != nil) { return nil, err } traitIsDiscreteOrNumeric := traitObject.DiscreteOrNumeric if (traitIsDiscreteOrNumeric != "Numeric"){ return nil, errors.New("setViewModelTestingNumericTraitResultsPage called with non-discrete trait: " + traitName) } probabilityMinimumRange := 1 for { if (probabilityMinimumRange == 100){ break } getProbabilityMaximumRange := func()int{ if (probabilityMinimumRange == 90){ return 100 } probabilityMaximumRange := probabilityMinimumRange + 9 return probabilityMaximumRange } probabilityMaximumRange := getProbabilityMaximumRange() probabilityMinimumRangeString := helpers.ConvertIntToString(probabilityMinimumRange) probabilityMaximumRangeString := helpers.ConvertIntToString(probabilityMaximumRange) probabilityOfCorrectPredictionRangeFormatted := probabilityMinimumRangeString + "% - " + probabilityMaximumRangeString + "%" probabilityOfCorrectPredictionRangeLabel := getBoldLabelCentered(probabilityOfCorrectPredictionRangeFormatted) // We use the below variables to sum up the accuracy distances so we can average them predictionAccuracyDistancesSum_0to33 := float64(0) distancesCount_0to33 := 0 predictionAccuracyDistancesSum_34to66 := float64(0) distancesCount_34to66 := 0 predictionAccuracyDistancesSum_67to100 := float64(0) distancesCount_67to100 := 0 for traitOutcomeInfo, traitPredictionAccuracyRangesMap := range traitAccuracyInfoMap{ percentageOfLociTested := traitOutcomeInfo.PercentageOfLociTested for percentageCorrect, distance := range traitPredictionAccuracyRangesMap{ if (percentageCorrect < probabilityMinimumRange || percentageCorrect > probabilityMaximumRange){ continue } if (percentageOfLociTested <= 33){ predictionAccuracyDistancesSum_0to33 += distance distancesCount_0to33 += 1 } else if (percentageOfLociTested > 33 && percentageOfLociTested <= 66){ predictionAccuracyDistancesSum_34to66 += distance distancesCount_34to66 += 1 } else { predictionAccuracyDistancesSum_67to100 += distance distancesCount_67to100 += 1 } } } getAverageAccuracyText := func(distancesSum float64, distancesCount int)string{ if (distancesCount == 0){ return "Unknown" } averageDistance := distancesSum/float64(distancesCount) averageDistanceString := helpers.ConvertFloat64ToStringRounded(averageDistance, 1) //TODO: Retrieve units from traits package? result := "+/- " + averageDistanceString + " centimeters" return result } averageDistanceText_0to33 := getAverageAccuracyText(predictionAccuracyDistancesSum_0to33, distancesCount_0to33) averageDistanceText_34to66 := getAverageAccuracyText(predictionAccuracyDistancesSum_34to66, distancesCount_34to66) averageDistanceText_67to100 := getAverageAccuracyText(predictionAccuracyDistancesSum_67to100, distancesCount_67to100) averageDistanceLabel_0to33 := getBoldLabelCentered(averageDistanceText_0to33) averageDistanceLabel_34to66 := getBoldLabelCentered(averageDistanceText_34to66) averageDistanceLabel_67to100 := getBoldLabelCentered(averageDistanceText_67to100) probabilityOfCorrectPredictionColumn.Add(probabilityOfCorrectPredictionRangeLabel) accuracyRangeColumn_0to33.Add(averageDistanceLabel_0to33) accuracyRangeColumn_34to66.Add(averageDistanceLabel_34to66) accuracyRangeColumn_67to100.Add(averageDistanceLabel_67to100) probabilityOfCorrectPredictionColumn.Add(widget.NewSeparator()) accuracyRangeColumn_0to33.Add(widget.NewSeparator()) accuracyRangeColumn_34to66.Add(widget.NewSeparator()) accuracyRangeColumn_67to100.Add(widget.NewSeparator()) if (probabilityMinimumRange == 1){ probabilityMinimumRange = 10 } else { probabilityMinimumRange += 10 } } resultsGrid := container.NewHBox(layout.NewSpacer(), probabilityOfCorrectPredictionColumn, accuracyRangeColumn_0to33, accuracyRangeColumn_34to66, accuracyRangeColumn_67to100, layout.NewSpacer()) return resultsGrid, nil } resultsGrid, err := getResultsGrid() if (err != nil){ setErrorEncounteredPage(window, err, func(){setHomePage(window)}) return } page := container.NewVBox(title, exitButton, widget.NewSeparator(), description1, widget.NewSeparator(), traitNameRow, widget.NewSeparator(), description2, description3, widget.NewSeparator(), resultsGrid) pageScrollable := container.NewVScroll(page) window.SetContent(pageScrollable) } // This function returns a list of training data and testing data filepaths for a trait. //Outputs: // -[]string: Sorted list of training data filepaths // -[]string: Unsorted list of testing data filepaths // -error func getTrainingAndTestingDataFilepathLists(traitName string)([]string, []string, error){ if (traitName != "Eye Color" && traitName != "Lactose Tolerance" && traitName != "Height"){ return nil, nil, errors.New("getTrainingAndTestingDataFilepathLists called with invalid traitName: " + traitName) } traitNameWithoutWhitespaces := strings.ReplaceAll(traitName, " ", "") trainingDataFolderpath := goFilepath.Join("./TrainingData/", traitNameWithoutWhitespaces) filesList, err := os.ReadDir(trainingDataFolderpath) if (err != nil) { return nil, nil, err } // This map stores the file name for each training data trainingDataFilenamesMap := make(map[string]struct{}) for _, filesystemObject := range filesList{ filepathIsFolder := filesystemObject.IsDir() if (filepathIsFolder == true){ // Folder is corrupt return nil, nil, errors.New("Training data is corrupt for trait: " + traitName) } fileName := filesystemObject.Name() trainingDataFilenamesMap[fileName] = struct{}{} } numberOfTrainingDataFiles := len(trainingDataFilenamesMap) if (numberOfTrainingDataFiles == 0){ return nil, nil, errors.New("No training data exists for trait: " + traitName) } getNumberOfExpectedTrainingDatas := func()(int, error){ if (traitName == "Eye Color"){ return 113648, nil } else if (traitName == "Lactose Tolerance"){ return 24872, nil } else if (traitName == "Height"){ return 92281, nil } return 0, errors.New("Unknown traitName: " + traitName) } numberOfExpectedTrainingDatas, err := getNumberOfExpectedTrainingDatas() if (err != nil){ return nil, nil, err } if (numberOfTrainingDataFiles != numberOfExpectedTrainingDatas){ numberOfTrainingDataFilesString := helpers.ConvertIntToString(numberOfTrainingDataFiles) return nil, nil, errors.New(traitName + " quantity of training datas is unexpected: " + numberOfTrainingDataFilesString) } // We sort the training data to be in a deterministically random order // This allows us to train the neural network in the same order each time // We do this so we can generate deterministic models which are identical byte-for-byte // We have to set aside 200 user's training datas for testing the neural network // // We have to remove them per-user because each user has 110 training datas. // Otherwise, we would be training and testing on data from the same users. // We need to test with users that the models were never trained upon. // First we extract the user identifiers from the data userIdentifiersMap := make(map[int]struct{}) for trainingDataFilename, _ := range trainingDataFilenamesMap{ // Example filepath format: "User4680_TrainingData_89.gob" trimmedFilename := strings.TrimPrefix(trainingDataFilename, "User") userIdentifierString, _, underscoreExists := strings.Cut(trimmedFilename, "_") if (underscoreExists == false){ return nil, nil, errors.New("Invalid trainingData filename: " + trainingDataFilename) } userIdentifier, err := helpers.ConvertStringToInt(userIdentifierString) if (err != nil){ return nil, nil, errors.New("Invalid trainingData filename: " + trainingDataFilename) } userIdentifiersMap[userIdentifier] = struct{}{} } userIdentifiersList := helpers.GetListOfMapKeys(userIdentifiersMap) // We sort the user identifiers list in ascending order slices.Sort(userIdentifiersList) // Now we deterministically randomize the order of the user identifiers list pseudorandomNumberGenerator := mathRand.New(mathRand.NewPCG(1, 2)) pseudorandomNumberGenerator.Shuffle(len(userIdentifiersList), func(i int, j int){ userIdentifiersList[i], userIdentifiersList[j] = userIdentifiersList[j], userIdentifiersList[i] }) trainingSetFilepathsList := make([]string, 0) testingSetFilepathsList := make([]string, 0) numberOfUsers := len(userIdentifiersList) if (numberOfUsers < 250){ return nil, nil, errors.New("Too few training data examples for trait: " + traitName) } // We use 200 users for testing (validation), so we don't train using them numberOfTrainingUsers := numberOfUsers - 200 for index, userIdentifier := range userIdentifiersList{ // Example filepath format: "User4680_TrainingData_89.gob" userIdentifierString := helpers.ConvertIntToString(userIdentifier) trainingDataFilenamePrefix := "User" + userIdentifierString + "_TrainingData_" for k:=1; k <= 110; k++{ kString := helpers.ConvertIntToString(k) trainingDataFilename := trainingDataFilenamePrefix + kString + ".gob" _, fileExists := trainingDataFilenamesMap[trainingDataFilename] if (fileExists == false){ // Some training datas don't exist due to how training datas are randomly created // Sometimes, no alleles exist, so we skip creating the training data continue } trainingDataFilepath := goFilepath.Join(trainingDataFolderpath, trainingDataFilename) if (index < numberOfTrainingUsers){ trainingSetFilepathsList = append(trainingSetFilepathsList, trainingDataFilepath) } else { testingSetFilepathsList = append(testingSetFilepathsList, trainingDataFilepath) } } } return trainingSetFilepathsList, testingSetFilepathsList, nil } // We use this to define a custom fyne theme // We are only overriding the foreground color to pure black type customTheme struct{ defaultTheme fyne.Theme } func getCustomFyneTheme()fyne.Theme{ standardThemeObject := theme.LightTheme() newTheme := customTheme{ defaultTheme: standardThemeObject, } return newTheme } // This function is used to define our custom fyne themes // It changes a few default colors, while leaving all other colors the same as the default theme func (input customTheme)Color(colorName fyne.ThemeColorName, variant fyne.ThemeVariant)color.Color{ switch colorName{ case theme.ColorNameForeground:{ newColor := color.Black return newColor } case theme.ColorNameSeparator:{ // This is the color used for separators newColor := color.Black return newColor } case theme.ColorNameInputBackground:{ // This color is used for the background of input elements such as text entries newColor, err := imagery.GetColorObjectFromColorCode("b3b3b3") if (err == nil){ return newColor } } case theme.ColorNameButton:{ // This is the color used for buttons newColor, err := imagery.GetColorObjectFromColorCode("d8d8d8") if (err == nil){ return newColor } } case theme.ColorNamePlaceHolder:{ // This is the color used for text newColor, err := imagery.GetColorObjectFromColorCode("4d4d4d") if (err == nil){ return newColor } } } // We will use the default color for this theme return input.defaultTheme.Color(colorName, variant) } // Our custom themes change nothing about the default theme fonts func (input customTheme)Font(style fyne.TextStyle)fyne.Resource{ themeFont := input.defaultTheme.Font(style) return themeFont } // Our custom themes change nothing about the default theme icons func (input customTheme)Icon(iconName fyne.ThemeIconName)fyne.Resource{ themeIcon := input.defaultTheme.Icon(iconName) return themeIcon } func (input customTheme)Size(name fyne.ThemeSizeName)float32{ themeSize := input.defaultTheme.Size(name) if (name == theme.SizeNameText){ // After fyne v2.3.0, text labels are no longer the same height as buttons // We increase the text size so that a text label is the same height as a button // We need to increase text size because we are creating grids by creating multiple VBoxes, and connecting them with an HBox // // If we could create grids in a different way, we could avoid having to do this // Example: Create a new grid type: container.NewThinGrid? // -The columns will only be as wide as the the widest element within them // -We can add separators between each row (grid.ShowRowLines = true) or between columns (grid.ShowColumnLines = true) // -We can add borders (grid.ShowTopBorder = true, grid.ShowBottomBorder = true, grid.ShowLeftBorder = true, grid.ShowRightBorder = true) // Using a different grid type is the solution we need to eventually use // Then, we can show the user an option to increase the text size globally, and all grids will still render correctly result := themeSize * 1.08 return result } return themeSize }