|
@@ -6,12 +6,22 @@ require "open3"
|
|
|
|
|
|
|
|
module Http
|
|
module Http
|
|
|
BASE_HEADERS = {
|
|
BASE_HEADERS = {
|
|
|
- "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
|
|
|
|
|
- "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
|
|
|
- "Accept-Language" => "en-AU,en;q=0.9",
|
|
|
|
|
- # keep identity to avoid manual gzip handling; servers still work with this
|
|
|
|
|
- "Accept-Encoding" => "identity",
|
|
|
|
|
- "Connection" => "keep-alive"
|
|
|
|
|
|
|
+ "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
|
|
|
|
|
+ "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
|
|
|
+ "Accept-Language" => "en-AU,en;q=0.9",
|
|
|
|
|
+ # identity avoids manual gzip handling; servers still respond correctly
|
|
|
|
|
+ "Accept-Encoding" => "identity",
|
|
|
|
|
+ "Connection" => "keep-alive",
|
|
|
|
|
+ # Browser fingerprint headers — modern WAFs check these to distinguish
|
|
|
|
|
+ # real browsers from scripts. Omitting them triggers 403/challenge pages.
|
|
|
|
|
+ "Upgrade-Insecure-Requests" => "1",
|
|
|
|
|
+ "Sec-Fetch-Dest" => "document",
|
|
|
|
|
+ "Sec-Fetch-Mode" => "navigate",
|
|
|
|
|
+ "Sec-Fetch-Site" => "none",
|
|
|
|
|
+ "Sec-Fetch-User" => "?1",
|
|
|
|
|
+ "sec-ch-ua" => '"Chromium";v="127", "Not)A;Brand";v="99", "Google Chrome";v="127"',
|
|
|
|
|
+ "sec-ch-ua-mobile" => "?0",
|
|
|
|
|
+ "sec-ch-ua-platform" => '"Windows"',
|
|
|
}.freeze
|
|
}.freeze
|
|
|
|
|
|
|
|
def self.merge_set_cookie!(jar, response)
|
|
def self.merge_set_cookie!(jar, response)
|
|
@@ -94,6 +104,14 @@ def self.get(url, headers: {}, tries: 4, referer: nil)
|
|
|
"-A", BASE_HEADERS["User-Agent"],
|
|
"-A", BASE_HEADERS["User-Agent"],
|
|
|
"-H", "Accept: #{BASE_HEADERS["Accept"]}",
|
|
"-H", "Accept: #{BASE_HEADERS["Accept"]}",
|
|
|
"-H", "Accept-Language: #{BASE_HEADERS["Accept-Language"]}",
|
|
"-H", "Accept-Language: #{BASE_HEADERS["Accept-Language"]}",
|
|
|
|
|
+ "-H", "Upgrade-Insecure-Requests: 1",
|
|
|
|
|
+ "-H", "Sec-Fetch-Dest: document",
|
|
|
|
|
+ "-H", "Sec-Fetch-Mode: navigate",
|
|
|
|
|
+ "-H", "Sec-Fetch-Site: same-origin",
|
|
|
|
|
+ "-H", "Sec-Fetch-User: ?1",
|
|
|
|
|
+ "-H", "sec-ch-ua: #{BASE_HEADERS["sec-ch-ua"]}",
|
|
|
|
|
+ "-H", "sec-ch-ua-mobile: ?0",
|
|
|
|
|
+ "-H", "sec-ch-ua-platform: #{BASE_HEADERS["sec-ch-ua-platform"]}",
|
|
|
"-e", ref,
|
|
"-e", ref,
|
|
|
uri.to_s
|
|
uri.to_s
|
|
|
)
|
|
)
|
