scrapychan/main.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

// Global HTTP client with timeout
var httpClient = &http.Client{
	Timeout: 30 * time.Second,
}

// PostData represents a 4chan post
type PostData struct {
	No  float64 `json:"no"`
	Tim float64 `json:"tim"`
	Ext string  `json:"ext"`
}

// ThreadData represents the JSON structure from 4chan API
type ThreadData struct {
	Posts []PostData `json:"posts"`
}

func writeDataToDisk(dest string, board string, verlog bool, post PostData, cdnresbody []byte) error {
	if cdnresbody == nil {
		return fmt.Errorf("no data to write")
	}

	filename := fmt.Sprintf("%s-%d%s", board, int64(post.Tim), post.Ext)
	filepath := filepath.Join(dest, filename)

	err := os.WriteFile(filepath, cdnresbody, 0664)
	if err != nil {
		return fmt.Errorf("failed to write file %s: %w", filepath, err)
	}

	if verlog {
		log.Printf("Successfully wrote image/video data to disk: %s", filename)
	}
	return nil
}

func getPostData(post PostData, board string, verlog bool) ([]byte, error) {
	if post.Ext == "" {
		if verlog {
			log.Printf("Post %d didn't include an image or video", int64(post.No))
		}
		return nil, nil
	}

	cdnurlstr := fmt.Sprintf("https://i.4cdn.org/%s/%d%s", board, int64(post.Tim), post.Ext)

	cdnres, err := httpClient.Get(cdnurlstr)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch media from %s: %w", cdnurlstr, err)
	}
	defer cdnres.Body.Close()

	if cdnres.StatusCode > 299 {
		return nil, fmt.Errorf("response failed with status code: %d for %s", cdnres.StatusCode, cdnurlstr)
	}

	if verlog {
		log.Printf("Got image/video %d%s data", int64(post.Tim), post.Ext)
	}

	cdnresbody, err := io.ReadAll(cdnres.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response body: %w", err)
	}

	if verlog {
		log.Println("Successfully got data from response body")
	}

	return cdnresbody, nil
}

func validateURL(url string) error {
	if url == "" {
		return fmt.Errorf("no thread URL provided")
	}

	if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "http://") {
		return fmt.Errorf("invalid URL format")
	}

	parts := strings.Split(url, "/")
	if len(parts) < 4 {
		return fmt.Errorf("invalid thread URL format")
	}

	return nil
}

func extractBoard(url string) string {
	parts := strings.Split(url, "/")
	if len(parts) >= 4 {
		return parts[3]
	}
	return ""
}

func fetchThreadData(url string, verlog bool) (*ThreadData, error) {
	apiURL := url + ".json"

	res, err := httpClient.Get(apiURL)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch thread data from %s: %w", apiURL, err)
	}
	defer res.Body.Close()

	if res.StatusCode > 299 {
		return nil, fmt.Errorf("API response failed with status code: %d", res.StatusCode)
	}

	if verlog {
		log.Println("Got thread data")
	}

	resbody, err := io.ReadAll(res.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read API response body: %w", err)
	}

	if verlog {
		log.Println("Got body of API response")
	}

	var jdata ThreadData
	if err := json.Unmarshal(resbody, &jdata); err != nil {
		return nil, fmt.Errorf("failed to unmarshal API response: %w", err)
	}

	if verlog {
		log.Println("Unmarshaled API response body")
	}

	return &jdata, nil
}

func main() {
	// Setting up command flags
	wdpath, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get working directory: %v", err)
	}

	url := flag.String("u", "", "The url of the 4chan thread")
	dest := flag.String("o", wdpath, "Target dir of the content")
	verlog := flag.Bool("v", false, "Set logging to verbose")

	flag.Parse()

	// Validate flags
	if err := validateURL(*url); err != nil {
		log.Fatalf("Invalid URL: %v", err)
	}

	// Getting the boardname
	board := extractBoard(*url)
	if board == "" {
		log.Fatal("Failed to extract board name from URL")
	}

	// Fetch thread data
	jdata, err := fetchThreadData(*url, *verlog)
	if err != nil {
		log.Fatalf("Failed to fetch thread data: %v", err)
	}

	// Ensure destination directory exists
	if err := os.MkdirAll(*dest, 0755); err != nil {
		log.Fatalf("Failed to create destination directory: %v", err)
	}

	var wg sync.WaitGroup

	// Iterating the posts from JSON data
	for _, post := range jdata.Posts {
		wg.Add(1)

		go func(post PostData) {
			defer wg.Done()

			if postdata, err := getPostData(post, board, *verlog); err != nil {
				log.Printf("Error processing post %d: %v", int64(post.No), err)
				return
			} else if postdata != nil {
				if err := writeDataToDisk(*dest, board, *verlog, post, postdata); err != nil {
					log.Printf("Error writing post %d to disk: %v", int64(post.No), err)
				}
			}
		}(post)
	}

	wg.Wait()

	log.Println("DONE!!!")
}