How to verify bulk domain/url list for 404 using threading and java

Email Marketing Email Verifications Technicals

You can you the below code . Just download a IDE called netbean create a project with maven then paste in the below  code click on play.  https://netbeans.apache.org/download/index.html

The below code uses threading to check if the url that it read from the file Domain_List.txt is valid or not.


package com.mycompany.mavenproject1;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Arrays;
import java.util.Hashtable;
import java.util.Scanner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.InitialDirContext;
import javax.naming.NamingException;
import java.util.Arrays;
import java.util.Comparator;
import org.xbill.DNS.*;

/**
*
* @author Andrew
*/
public class TestJava {

private static final int MYTHREADS = 500;

public static class MyRunnable implements Runnable {

private final String url;
volatile static int done = 0;
volatile static Hashtable<String, String> my_results = new Hashtable<String, String>();
static RandomAccessFile file = new RandomAccessFile(“C:\\jv_result_5.txt”, “rw”);
final int bufferSize = 512;
final AtomicInteger byteCounter = new AtomicInteger(0);

MyRunnable(String url) {
this.url = url;
}

@Override
public void run() {

String result = “”;
int code = 200;
try {
URL siteURL = new URL(url);
HttpURLConnection connection = (HttpURLConnection) siteURL.openConnection();
connection.setRequestMethod(“GET”);
connection.setConnectTimeout(3000);
connection.connect();

code = connection.getResponseCode();
if (code == 200) {
my_results.put(url, String.valueOf(code));
} else {
my_results.put(url, String.valueOf(code));
}
} catch (Exception e) {
my_results.put(url, String.valueOf(e.getMessage()));

}

String toOut = url + “\tStatus:” + String.valueOf(code);
String toPrint = toOut + “\n”;
System.out.println(toOut);
try {
file.write(toPrint.getBytes()); //exception handle
} catch (IOException ex) {
Logger.getLogger(TestJava.class.getName()).log(Level.SEVERE, null, ex);
}
}

}

}

public static void main(String args[]) throws Exception {

ExecutorService executor = Executors.newFixedThreadPool(MYTHREADS);
String[] hostList = {};
FileInputStream fis = new FileInputStream(“C:\\Domain_List.txt”);
Scanner sc = new Scanner(fis); //file to be scanned
//returns true if there is another line to read
int cnt = 0;
while (sc.hasNextLine()) {
//System.out.println(sc.nextLine()); //returns the line that was skipped
String url = sc.nextLine();
Runnable worker = new MyRunnable(url);
executor.execute(worker);
}
sc.close(); //closes the scanner

executor.shutdown();
// Wait until all threads are finish
while (!executor.isTerminated()) {

}
System.out.println(“\nFinished all threads”);
}

}