Changed concurrency to use a waitgroup
This commit is contained in:
24
main.go
24
main.go
@@ -10,9 +10,9 @@ import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
|
||||
|
||||
// Save the mediadata to file
|
||||
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
|
||||
if err != nil {
|
||||
@@ -22,7 +22,7 @@ func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]i
|
||||
}
|
||||
}
|
||||
|
||||
func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) {
|
||||
func getPostData(post map[string]interface{}, board string, verlog *bool) []byte {
|
||||
// Check if post contains media (Video or Image)
|
||||
if post["ext"] != nil {
|
||||
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
|
||||
@@ -48,11 +48,12 @@ func getPostData(post map[string]interface{}, board string, verlog *bool, dataCh
|
||||
} else if (*verlog) {
|
||||
log.Println("Successfully got data from responds body")
|
||||
}
|
||||
dataChannel <- cdnresbody
|
||||
return cdnresbody
|
||||
|
||||
} else if (*verlog) {
|
||||
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -112,17 +113,24 @@ func main () {
|
||||
log.Println("Unmarsheled API responsebody")
|
||||
}
|
||||
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Iterating the posts from JSON data
|
||||
for _, v := range jdata["posts"].([]interface{}) {
|
||||
post := v.(map[string]interface{})
|
||||
|
||||
postDataChannel := make(chan []byte)
|
||||
|
||||
go getPostData(post, board, verlog, postDataChannel)
|
||||
|
||||
go writeDataToDisk(dest, board, verlog, post, <-postDataChannel)
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if postdata := getPostData(post, board, verlog); postdata != nil {
|
||||
writeDataToDisk(dest, board, verlog, post, postdata)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
log.Println("DONE!!!")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user