Doc/scrapychan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Post request and writing data to disk is now async

Browse Source
This commit is contained in:
Doc
2024-03-19 13:50:34 +01:00
parent 44ca443a7c
commit 6b415b0ef1
1 changed files with 48 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
main.go
View File

@@ -11,6 +11,50 @@ import (
"strconv"
"strings"
)
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
// Save the mediadata to file
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully wrote image/video data to disk")
}
}
func getPostData(post map[string]interface{}, board string, verlog *bool, dataChannel chan []byte) {
// Check if post contains media (Video or Image)
if post["ext"] != nil {
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
// Requesting the media from CDN
cdnres, err := http.Get(cdnurlstr)
if err != nil {
log.Fatal(err)
}
// Check if respons was valid
if cdnres.StatusCode > 299 {
log.Fatalf("Response failed with status code: %d and\n", cdnres.StatusCode)
} else if (*verlog) {
log.Println("Got image/video " + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + " data")
}
// Read data form respons
cdnresbody, err := io.ReadAll(cdnres.Body)
cdnres.Body.Close()
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully got data from responds body")
}
dataChannel <- cdnresbody
} else if (*verlog) {
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
}
}
func main () {
// Setting up command flags
@@ -72,47 +116,12 @@ func main () {
for _, v := range jdata["posts"].([]interface{}) {
post := v.(map[string]interface{})
// Check if post contains media (Video or Image)
if post["ext"] != nil {
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
// Requesting the media from CDN
cdnres, err := http.Get(cdnurlstr)
if err != nil {
log.Fatal(err)
}
// Check if respons was valid
if res.StatusCode > 299 {
log.Fatalf("Response failed with status code: %d and\n", res.StatusCode)
} else if (*verlog) {
log.Println("Got image/video " + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + " data")
}
// Read data form respons
cdnresbody, err := io.ReadAll(cdnres.Body)
cdnres.Body.Close()
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully got data from responds body")
}
postDataChannel := make(chan []byte)
// Save the mediadata to file
err = os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully wrote image/video data to disk")
}
go getPostData(post, board, verlog, postDataChannel)
go writeDataToDisk(dest, board, verlog, post, <-postDataChannel)
} else if (*verlog) {
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
}
}
log.Println("DONE!!!")