Changed concurrency to use a waitgroup

This commit is contained in:
Doc
2024-03-20 09:51:29 +01:00
parent 6b415b0ef1
commit 3bf0dd812f

24
main.go
View File

@@ -10,9 +10,9 @@ import (
"os"
"strconv"
"strings"
"sync"
)
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
// Save the mediadata to file
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
if err != nil {
@@ -22,7 +22,7 @@ func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]i
}
}
func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) {
func getPostData(post map[string]interface{}, board string, verlog *bool) []byte {
// Check if post contains media (Video or Image)
if post["ext"] != nil {
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
@@ -48,11 +48,12 @@ func getPostData(post map[string]interface{}, board string, verlog *bool, dataCh
} else if (*verlog) {
log.Println("Successfully got data from responds body")
}
dataChannel <- cdnresbody
return cdnresbody
} else if (*verlog) {
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
}
return nil
}
@@ -112,17 +113,24 @@ func main () {
log.Println("Unmarsheled API responsebody")
}
var wg sync.WaitGroup
// Iterating the posts from JSON data
for _, v := range jdata["posts"].([]interface{}) {
post := v.(map[string]interface{})
postDataChannel := make(chan []byte)
go getPostData(post, board, verlog, postDataChannel)
go writeDataToDisk(dest, board, verlog, post, <-postDataChannel)
wg.Add(1)
go func() {
defer wg.Done()
if postdata := getPostData(post, board, verlog); postdata != nil {
writeDataToDisk(dest, board, verlog, post, postdata)
}
}()
}
wg.Wait()
log.Println("DONE!!!")