enabling gos comression negotiation

internal/scraper/session.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"log"
+	"mime/multipart"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
@@ -39,7 +40,6 @@ func NewSession() *Session {
 			"User-Agent":                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
 			"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
 			"Accept-Language":           "en-US,en;q=0.5",
-			"Accept-Encoding":           "gzip, deflate, br",
 			"Connection":                "keep-alive",
 			"Upgrade-Insecure-Requests": "1",
 		},
@@ -87,22 +87,26 @@ func (s *Session) Get(url string, headers ...map[string]string) (*http.Response,
 }
 
 // Post performs a POST request with form data
-func (s *Session) Post(url string, data map[string]string, headers ...map[string]string) (*http.Response, error) {
+func (s *Session) PostMultipartForm(url string, data map[string]string, headers ...map[string]string) (*http.Response, error) {
 	fullURL := s.buildURL(url)
 
-	// This is the corrected line - url.Values is from net/url package
-	formData := make(url.Values)
-	for key, value := range data {
-		formData.Add(key, value)
+	var requestBody bytes.Buffer
+	writer := multipart.NewWriter(&requestBody)
+	for k, v := range data {
+		err := writer.WriteField(k, v)
+		if err != nil {
+			return nil, err
+		}
 	}
+	writer.Close()
 
-	req, err := http.NewRequest("POST", fullURL, bytes.NewBufferString(formData.Encode()))
+	req, err := http.NewRequest("POST", fullURL, &requestBody)
 	if err != nil {
 		return nil, err
 	}
 
 	s.setDefaultHeaders(req)
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Content-Type", writer.FormDataContentType())
 
 	// Add any additional headers provided
 	if len(headers) > 0 {