package main import ( "encoding/json" "flag" "fmt" "io" "log" "net/http" "os" "path/filepath" "strings" "sync" "time" ) // Global HTTP client with timeout var httpClient = &http.Client{ Timeout: 30 * time.Second, } // PostData represents a 4chan post type PostData struct { No float64 `json:"no"` Tim float64 `json:"tim"` Ext string `json:"ext"` } // ThreadData represents the JSON structure from 4chan API type ThreadData struct { Posts []PostData `json:"posts"` } func writeDataToDisk(dest string, board string, verlog bool, post PostData, cdnresbody []byte) error { if cdnresbody == nil { return fmt.Errorf("no data to write") } filename := fmt.Sprintf("%s-%d%s", board, int64(post.Tim), post.Ext) filepath := filepath.Join(dest, filename) err := os.WriteFile(filepath, cdnresbody, 0664) if err != nil { return fmt.Errorf("failed to write file %s: %w", filepath, err) } if verlog { log.Printf("Successfully wrote image/video data to disk: %s", filename) } return nil } func getPostData(post PostData, board string, verlog bool) ([]byte, error) { if post.Ext == "" { if verlog { log.Printf("Post %d didn't include an image or video", int64(post.No)) } return nil, nil } cdnurlstr := fmt.Sprintf("https://i.4cdn.org/%s/%d%s", board, int64(post.Tim), post.Ext) cdnres, err := httpClient.Get(cdnurlstr) if err != nil { return nil, fmt.Errorf("failed to fetch media from %s: %w", cdnurlstr, err) } defer cdnres.Body.Close() if cdnres.StatusCode > 299 { return nil, fmt.Errorf("response failed with status code: %d for %s", cdnres.StatusCode, cdnurlstr) } if verlog { log.Printf("Got image/video %d%s data", int64(post.Tim), post.Ext) } cdnresbody, err := io.ReadAll(cdnres.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } if verlog { log.Println("Successfully got data from response body") } return cdnresbody, nil } func validateURL(url string) error { if url == "" { return fmt.Errorf("no thread URL provided") } if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") { return fmt.Errorf("invalid URL format") } parts := strings.Split(url, "/") if len(parts) < 4 { return fmt.Errorf("invalid thread URL format") } return nil } func extractBoard(url string) string { parts := strings.Split(url, "/") if len(parts) >= 4 { return parts[3] } return "" } func fetchThreadData(url string, verlog bool) (*ThreadData, error) { apiURL := url + ".json" res, err := httpClient.Get(apiURL) if err != nil { return nil, fmt.Errorf("failed to fetch thread data from %s: %w", apiURL, err) } defer res.Body.Close() if res.StatusCode > 299 { return nil, fmt.Errorf("API response failed with status code: %d", res.StatusCode) } if verlog { log.Println("Got thread data") } resbody, err := io.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("failed to read API response body: %w", err) } if verlog { log.Println("Got body of API response") } var jdata ThreadData if err := json.Unmarshal(resbody, &jdata); err != nil { return nil, fmt.Errorf("failed to unmarshal API response: %w", err) } if verlog { log.Println("Unmarshaled API response body") } return &jdata, nil } func main() { // Setting up command flags wdpath, err := os.Getwd() if err != nil { log.Fatalf("Failed to get working directory: %v", err) } url := flag.String("u", "", "The url of the 4chan thread") dest := flag.String("o", wdpath, "Target dir of the content") verlog := flag.Bool("v", false, "Set logging to verbose") flag.Parse() // Validate flags if err := validateURL(*url); err != nil { log.Fatalf("Invalid URL: %v", err) } // Getting the boardname board := extractBoard(*url) if board == "" { log.Fatal("Failed to extract board name from URL") } // Fetch thread data jdata, err := fetchThreadData(*url, *verlog) if err != nil { log.Fatalf("Failed to fetch thread data: %v", err) } // Ensure destination directory exists if err := os.MkdirAll(*dest, 0755); err != nil { log.Fatalf("Failed to create destination directory: %v", err) } var wg sync.WaitGroup // Iterating the posts from JSON data for _, post := range jdata.Posts { wg.Add(1) go func(post PostData) { defer wg.Done() if postdata, err := getPostData(post, board, *verlog); err != nil { log.Printf("Error processing post %d: %v", int64(post.No), err) return } else if postdata != nil { if err := writeDataToDisk(*dest, board, *verlog, post, postdata); err != nil { log.Printf("Error writing post %d to disk: %v", int64(post.No), err) } } }(post) } wg.Wait() log.Println("DONE!!!") }