kamma/cz-basketball-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added gzip support

Browse Source
This commit is contained in:
Radek Davidek 2025-11-02 16:24:56 +01:00
parent d40c6cf403
commit 5647dbd783
2 changed files with 31 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
logs/basketball-scraper.log
View File

@ -8,3 +8,18 @@
2025-11-02 16:01:10.568 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Handling API request: /api/matches?date=2025-11-02 2025-11-02 16:01:10.568 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Handling API request: /api/matches?date=2025-11-02
2025-11-02 16:01:10.570 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Fetching initial cookies from cz.basketball 2025-11-02 16:01:10.570 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Fetching initial cookies from cz.basketball
2025-11-02 16:01:11.451 [pool-3-thread-2] INFO cz.kamma.czb.BasketballServer - Data fetched successfully for date: 2025-11-02 2025-11-02 16:01:11.451 [pool-3-thread-2] INFO cz.kamma.czb.BasketballServer - Data fetched successfully for date: 2025-11-02
2025-11-02 16:23:15.445 [main] INFO cz.kamma.czb.BasketballServer - Starting Basketball Server...
2025-11-02 16:23:15.700 [main] INFO cz.kamma.czb.BasketballServer - Server running at http://localhost:8000/
2025-11-02 16:23:24.626 [pool-3-thread-1] DEBUG cz.kamma.czb.BasketballServer - Handling Web request: /
2025-11-02 16:23:24.713 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Handling API request: /api/matches?date=2025-11-02
2025-11-02 16:23:24.715 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Fetching initial cookies from cz.basketball
2025-11-02 16:23:25.592 [pool-3-thread-2] INFO cz.kamma.czb.BasketballServer - Data fetched successfully for date: 2025-11-02
2025-11-02 16:24:29.839 [main] INFO cz.kamma.czb.BasketballServer - Starting Basketball Server...
2025-11-02 16:24:30.103 [main] INFO cz.kamma.czb.BasketballServer - Server running at http://localhost:8000/
2025-11-02 16:24:33.039 [pool-3-thread-1] DEBUG cz.kamma.czb.BasketballServer - Handling Web request: /
2025-11-02 16:24:33.100 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Handling API request: /api/matches?date=2025-11-02
2025-11-02 16:24:33.101 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Fetching initial cookies from cz.basketball
2025-11-02 16:24:33.801 [pool-3-thread-2] INFO cz.kamma.czb.BasketballServer - Initial request encoding: gzip
2025-11-02 16:24:33.859 [pool-3-thread-2] INFO cz.kamma.czb.BasketballServer - Main request encoding: gzip
2025-11-02 16:24:33.871 [pool-3-thread-2] DEBUG cz.kamma.czb.BasketballServer - Using GZIP decompression for response
2025-11-02 16:24:33.972 [pool-3-thread-2] INFO cz.kamma.czb.BasketballServer - Data fetched successfully for date: 2025-11-02

17
src/main/java/cz/kamma/czb/BasketballServer.java
View File

@ -48,6 +48,10 @@ public class BasketballServer {
URL initialUrl = new URL("https://cz.basketball"); URL initialUrl = new URL("https://cz.basketball");
HttpURLConnection initialCon = (HttpURLConnection) initialUrl.openConnection(); HttpURLConnection initialCon = (HttpURLConnection) initialUrl.openConnection();
initialCon.setRequestMethod("GET"); initialCon.setRequestMethod("GET");
initialCon.setRequestProperty("Accept-Encoding", "gzip");
String initialEncoding = initialCon.getContentEncoding();
log.info("Initial request encoding: {}", initialEncoding != null ? initialEncoding : "none");
// Get cookies from response headers // Get cookies from response headers
String cookies = ""; String cookies = "";
@ -73,9 +77,20 @@ public class BasketballServer {
URL url = new URL(urlString); URL url = new URL(urlString);
HttpURLConnection con = (HttpURLConnection) url.openConnection(); HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET"); con.setRequestMethod("GET");
con.setRequestProperty("Accept-Encoding", "gzip");
con.setRequestProperty("Cookie", cookies); con.setRequestProperty("Cookie", cookies);
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8)); // Handle potential gzipped response
InputStream inputStream = con.getInputStream();
String encoding = con.getContentEncoding();
log.info("Main request encoding: {}", encoding != null ? encoding : "none");
if ("gzip".equalsIgnoreCase(encoding)) {
inputStream = new java.util.zip.GZIPInputStream(inputStream);
log.debug("Using GZIP decompression for response");
}
BufferedReader in = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
StringBuilder content = new StringBuilder(); StringBuilder content = new StringBuilder();
String line; String line;
while ((line = in.readLine()) != null) content.append(line); while ((line = in.readLine()) != null) content.append(line);