Changed concurrency to use a waitgroup
This commit is contained in:
24
main.go
24
main.go
@@ -10,9 +10,9 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
|
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
|
||||||
|
|
||||||
// Save the mediadata to file
|
// Save the mediadata to file
|
||||||
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
|
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -22,7 +22,7 @@ func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]i
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) {
|
func getPostData(post map[string]interface{}, board string, verlog *bool) []byte {
|
||||||
// Check if post contains media (Video or Image)
|
// Check if post contains media (Video or Image)
|
||||||
if post["ext"] != nil {
|
if post["ext"] != nil {
|
||||||
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
|
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
|
||||||
@@ -48,11 +48,12 @@ func getPostData(post map[string]interface{}, board string, verlog *bool, dataCh
|
|||||||
} else if (*verlog) {
|
} else if (*verlog) {
|
||||||
log.Println("Successfully got data from responds body")
|
log.Println("Successfully got data from responds body")
|
||||||
}
|
}
|
||||||
dataChannel <- cdnresbody
|
return cdnresbody
|
||||||
|
|
||||||
} else if (*verlog) {
|
} else if (*verlog) {
|
||||||
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
|
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -112,18 +113,25 @@ func main () {
|
|||||||
log.Println("Unmarsheled API responsebody")
|
log.Println("Unmarsheled API responsebody")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
// Iterating the posts from JSON data
|
// Iterating the posts from JSON data
|
||||||
for _, v := range jdata["posts"].([]interface{}) {
|
for _, v := range jdata["posts"].([]interface{}) {
|
||||||
post := v.(map[string]interface{})
|
post := v.(map[string]interface{})
|
||||||
|
|
||||||
postDataChannel := make(chan []byte)
|
wg.Add(1)
|
||||||
|
|
||||||
go getPostData(post, board, verlog, postDataChannel)
|
|
||||||
|
|
||||||
go writeDataToDisk(dest, board, verlog, post, <-postDataChannel)
|
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if postdata := getPostData(post, board, verlog); postdata != nil {
|
||||||
|
writeDataToDisk(dest, board, verlog, post, postdata)
|
||||||
|
}
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
log.Println("DONE!!!")
|
log.Println("DONE!!!")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user