Doc/scrapychan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge pull request 'add-async-functionality' (#1) from add-async-functionality into master

Browse Source 
Reviewed-on: #1
This commit is contained in:
Doc
2024-03-20 10:01:37 +01:00
parent 44ca443a7c 3bf0dd812f
commit 59ef434f5b
1 changed files with 57 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
main.go
View File

@@ -10,7 +10,52 @@ import (
"os"
"strconv"
"strings"
"sync"
)
func writeDataToDisk(dest *string, board string, verlog *bool, post map[string]interface{}, cdnresbody []byte) {
// Save the mediadata to file
err := os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully wrote image/video data to disk")
}
}
func getPostData(post map[string]interface{}, board string, verlog *bool) []byte {
// Check if post contains media (Video or Image)
if post["ext"] != nil {
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
// Requesting the media from CDN
cdnres, err := http.Get(cdnurlstr)
if err != nil {
log.Fatal(err)
}
// Check if respons was valid
if cdnres.StatusCode > 299 {
log.Fatalf("Response failed with status code: %d and\n", cdnres.StatusCode)
} else if (*verlog) {
log.Println("Got image/video " + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + " data")
}
// Read data form respons
cdnresbody, err := io.ReadAll(cdnres.Body)
cdnres.Body.Close()
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully got data from responds body")
}
return cdnresbody
} else if (*verlog) {
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
}
return nil
}
func main () {
// Setting up command flags
@@ -67,54 +112,26 @@ func main () {
} else if (*verlog) {
log.Println("Unmarsheled API responsebody")
}
var wg sync.WaitGroup
// Iterating the posts from JSON data
for _, v := range jdata["posts"].([]interface{}) {
post := v.(map[string]interface{})
// Check if post contains media (Video or Image)
if post["ext"] != nil {
cdnurlstr := "https://i.4cdn.org/" + board + "/" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string)
// Requesting the media from CDN
cdnres, err := http.Get(cdnurlstr)
if err != nil {
log.Fatal(err)
}
// Check if respons was valid
if res.StatusCode > 299 {
log.Fatalf("Response failed with status code: %d and\n", res.StatusCode)
} else if (*verlog) {
log.Println("Got image/video " + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string) + " data")
}
// Read data form respons
cdnresbody, err := io.ReadAll(cdnres.Body)
cdnres.Body.Close()
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully got data from responds body")
}
wg.Add(1)
// Save the mediadata to file
err = os.WriteFile(*dest + "/" + board + "-" + strconv.Itoa(int(post["tim"].(float64))) + post["ext"].(string), cdnresbody, 0664 )
if err != nil {
log.Fatal(err)
} else if (*verlog) {
log.Println("Successfully wrote image/video data to disk")
go func() {
defer wg.Done()
if postdata := getPostData(post, board, verlog); postdata != nil {
writeDataToDisk(dest, board, verlog, post, postdata)
}
} else if (*verlog) {
log.Println("Post " + strconv.Itoa(int(post["no"].(float64))) + " didn't include a image or video")
}
}()
}
wg.Wait()
log.Println("DONE!!!")
}