Changed concurrency to use a waitgroup

This commit is contained in:
Doc
2024-03-20 09:51:29 +01:00
parent 6b415b0ef1
commit 3bf0dd812f

24
main.go
View File

@@ -10,9 +10,9 @@ import (
"os" "os"
"strconv" "strconv"
"strings" "strings"
"sync"
) )
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) { func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
// Save the mediadata to file // Save the mediadata to file
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 ) err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
if err != nil { if err != nil {
@@ -22,7 +22,7 @@ func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]i
} }
} }
func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) { func getPostData(post map[string]interface{}, board string, verlog *bool) []byte {
// Check if post contains media (Video or Image) // Check if post contains media (Video or Image)
if post["ext"] != nil { if post["ext"] != nil {
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
@@ -48,11 +48,12 @@ func getPostData(post map[string]interface{}, board string, verlog *bool, dataCh
} else if (*verlog) { } else if (*verlog) {
log.Println("Successfully got data from responds body") log.Println("Successfully got data from responds body")
} }
dataChannel <- cdnresbody return cdnresbody
} else if (*verlog) { } else if (*verlog) {
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video") log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
} }
return nil
} }
@@ -112,17 +113,24 @@ func main () {
log.Println("Unmarsheled API responsebody") log.Println("Unmarsheled API responsebody")
} }
var wg sync.WaitGroup
// Iterating the posts from JSON data // Iterating the posts from JSON data
for _, v := range jdata["posts"].([]interface{}) { for _, v := range jdata["posts"].([]interface{}) {
post := v.(map[string]interface{}) post := v.(map[string]interface{})
postDataChannel := make(chan []byte) wg.Add(1)
go getPostData(post, board, verlog, postDataChannel)
go writeDataToDisk(dest, board, verlog, post, <-postDataChannel)
go func() {
defer wg.Done()
if postdata := getPostData(post, board, verlog); postdata != nil {
writeDataToDisk(dest, board, verlog, post, postdata)
} }
}()
}
wg.Wait()
log.Println("DONE!!!") log.Println("DONE!!!")