From 6b415b0ef1348561192450ba23cfbdb3eb5de58e Mon Sep 17 00:00:00 2001 From: Doc Date: Tue, 19 Mar 2024 13:50:34 +0100 Subject: [PATCH 1/2] Post request and writing data to disk is now async --- main.go | 87 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/main.go b/main.go index 55d4bbc..149b060 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,50 @@ import ( "strconv" "strings" ) +func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) { + + // Save the mediadata to file + err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 ) + if err != nil { + log.Fatal(err) + } else if (*verlog) { + log.Println("Successfully wrote image/video data to disk") + } +} + +func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) { + // Check if post contains media (Video or Image) + if post["ext"] != nil { + cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + + // Requesting the media from CDN + cdnres, err := http.Get(cdnurlstr) + if err != nil { + log.Fatal(err) + } + + // Check if respons was valid + if cdnres.StatusCode > 299 { + log.Fatalf("Response failed with status code: %d and\n", cdnres.StatusCode) + } else if (*verlog) { + log.Println("Got image/video " + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + " data") + } + + // Read data form respons + cdnresbody, err := io.ReadAll(cdnres.Body) + cdnres.Body.Close() + if err != nil { + log.Fatal(err) + } else if (*verlog) { + log.Println("Successfully got data from responds body") + } + dataChannel <- cdnresbody + + } else if (*verlog) { + log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video") + } +} + func main () { // Setting up command flags @@ -72,47 +116,12 @@ func main () { for _, v := range jdata["posts"].([]interface{}) { post := v.(map[string]interface{}) - // Check if post contains media (Video or Image) - if post["ext"] != nil { - - - cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) - - // Requesting the media from CDN - cdnres, err := http.Get(cdnurlstr) - if err != nil { - log.Fatal(err) - } - - // Check if respons was valid - if res.StatusCode > 299 { - log.Fatalf("Response failed with status code: %d and\n", res.StatusCode) - } else if (*verlog) { - log.Println("Got image/video " + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + " data") - } - - // Read data form respons - cdnresbody, err := io.ReadAll(cdnres.Body) - cdnres.Body.Close() - if err != nil { - log.Fatal(err) - } else if (*verlog) { - log.Println("Successfully got data from responds body") - } - + postDataChannel := make(chan []byte) - // Save the mediadata to file - err = os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 ) - if err != nil { - log.Fatal(err) - } else if (*verlog) { - log.Println("Successfully wrote image/video data to disk") - } + go getPostData(post, board, verlog, postDataChannel) + + go writeDataToDisk(dest, board, verlog, post, <-postDataChannel) - } else if (*verlog) { - log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video") - } - } log.Println("DONE!!!") From 3bf0dd812f69be50c61706a8502549b2c8cb457d Mon Sep 17 00:00:00 2001 From: Doc Date: Wed, 20 Mar 2024 09:51:29 +0100 Subject: [PATCH 2/2] Changed concurrency to use a waitgroup --- main.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 149b060..d9f7bba 100644 --- a/main.go +++ b/main.go @@ -10,9 +10,9 @@ import ( "os" "strconv" "strings" + "sync" ) func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) { - // Save the mediadata to file err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 ) if err != nil { @@ -22,7 +22,7 @@ func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]i } } -func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) { +func getPostData(post map[string]interface{}, board string, verlog *bool) []byte { // Check if post contains media (Video or Image) if post["ext"] != nil { cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) @@ -48,11 +48,12 @@ func getPostData(post map[string]interface{}, board string, verlog *bool, dataCh } else if (*verlog) { log.Println("Successfully got data from responds body") } - dataChannel <- cdnresbody + return cdnresbody } else if (*verlog) { log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video") } + return nil } @@ -111,19 +112,26 @@ func main () { } else if (*verlog) { log.Println("Unmarsheled API responsebody") } + + + var wg sync.WaitGroup // Iterating the posts from JSON data for _, v := range jdata["posts"].([]interface{}) { post := v.(map[string]interface{}) - - postDataChannel := make(chan []byte) - - go getPostData(post, board, verlog, postDataChannel) - - go writeDataToDisk(dest, board, verlog, post, <-postDataChannel) - + + wg.Add(1) + + go func() { + defer wg.Done() + if postdata := getPostData(post, board, verlog); postdata != nil { + writeDataToDisk(dest, board, verlog, post, postdata) + } + }() } + wg.Wait() + log.Println("DONE!!!") }