Protocol util - add option to dump the content to a tmp file

Signed-off-by: Julien Nioche <julien@digitalpebble.com>
apache · Sep 26, 2023 · c630e61 · c630e61
1 parent bc21ebf
commit c630e61
Showing 1 changed file with 14 additions and 1 deletion.
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/AbstractHttpProtocol.java b/core/src/main/java/com/digitalpebble/stormcrawler/protocol/AbstractHttpProtocol.java
@@ -22,6 +22,8 @@
 import crawlercommons.robots.BaseRobotRules;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -31,6 +33,7 @@
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.Options;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.storm.Config;
 import org.apache.storm.utils.Utils;
@@ -205,6 +208,7 @@ protected static void main(AbstractHttpProtocol protocol, String[] args) throws
 
         Options options = new Options();
         options.addOption("f", true, "configuration file");
+        options.addOption("b", false, "dump binary content to temp file");
 
         CommandLineParser parser = new DefaultParser();
         CommandLine cmd = parser.parse(options, args);
@@ -214,6 +218,8 @@ protected static void main(AbstractHttpProtocol protocol, String[] args) throws
             ConfUtils.loadConf(confFile, conf);
         }
 
+        boolean binary = cmd.hasOption("b");
+
         protocol.configure(conf);
 
         Set<Runnable> threads = new HashSet<>();
@@ -258,7 +264,14 @@ public void run() {
                             .append(response.getContent().length)
                             .append("\n");
                     long timeFetching = System.currentTimeMillis() - start;
-                    stringB.append("fetched in : ").append(timeFetching).append(" msec");
+                    stringB.append("fetched in : ").append(timeFetching).append(" msec\n");
+
+                    if (binary) {
+                        Path p = Files.createTempFile("sc-protocol-", ".dump");
+                        FileUtils.writeByteArrayToFile(p.toFile(), response.getContent());
+                        stringB.append("dumped content to : ").append(p);
+                    }
+
                     System.out.println(stringB);
                 } catch (Exception e) {
                     e.printStackTrace();