import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; import java.util.Enumeration; import java.util.Hashtable; import java.util.List; /** * DuplicateFinder - Command line program to find duplicate files using MD5 hashing. * * Uses a Hashtable where MD5 hash (String) -> List of file paths (String). * Files sharing the same MD5 are duplicates (identical content). * * Usage: * java DuplicateFinder [options] <directory> * java DuplicateFinder --gui (launches Swing GUI) * * Options: * -r, --recursive Recurse into subdirectories (default: true) * --no-recursive Do not recurse * -h, --help Show help * --gui Launch Swing GUI * * Standalone CLI - can be used without GUI. * GUI wrapper is in DuplicateFinderGUI.java */ public class DuplicateFinder { private static final int BUFFER_SIZE = 8192; /** * Compute MD5 hash for a file and return as hex string. * @param file file to hash * @return hex MD5 string * @throws IOException if read fails */ public static String computeMD5(File file) throws IOException { MessageDigest md; try { md = MessageDigest.getInstance("MD5"); } catch (NoSuchAlgorithmException e) { throw new RuntimeException("MD5 not available", e); } InputStream is = null; try { is = new FileInputStream(file); byte[] buffer = new byte[BUFFER_SIZE]; int read; while ((read = is.read(buffer)) != -1) { md.update(buffer, 0, read); } } finally { if (is != null) { try { is.close(); } catch (IOException ignored) {} } } byte[] digest = md.digest(); StringBuilder sb = new StringBuilder(32); for (byte b : digest) { sb.append(String.format("%02x", b & 0xff)); } return sb.toString(); } /** * Collect all files under root. * @param root starting directory or file * @param recursive if true, walk subdirectories * @return list of files (not directories) */ public static List collectFiles(File root, boolean recursive) { List files = new ArrayList(); if (root == null || !root.exists()) { return files; } if (root.isFile()) { files.add(root); return files; } File[] children = root.listFiles(); if (children == null) { System.err.println("Warning: cannot list directory: " + root.getAbsolutePath()); return files; } for (File child : children) { if (child.isFile()) { files.add(child); } else if (child.isDirectory() && recursive) { // avoid following symlinks that cause cycles? isFile/isDirectory follows links but // we simply recurse; deep cycles unlikely on Windows files.addAll(collectFiles(child, true)); } else if (child.isDirectory() && !recursive) { // skip } } return files; } /** * Build Hashtable mapping MD5 hash -> list of file paths. * This is the core hashtable required by the specification. * * @param root directory to search * @param recursive recurse flag * @return Hashtable where key = md5 hex, value = list of absolute paths */ public static Hashtable> buildHashTable(File root, boolean recursive) { Hashtable> table = new Hashtable>(); List files = collectFiles(root, recursive); for (File f : files) { try { // skip zero-length? No, include - empty files duplicates too // skip unreadable if (!f.canRead()) { System.err.println("Warning: cannot read file: " + f.getAbsolutePath()); continue; } String md5 = computeMD5(f); List list = table.get(md5); if (list == null) { list = new ArrayList(); table.put(md5, list); } list.add(f.getAbsolutePath()); } catch (IOException ex) { System.err.println("Warning: failed to hash " + f.getAbsolutePath() + " : " + ex.getMessage()); } } return table; } /** * Filter hashtable to only entries with duplicates (more than one file per hash). * @param table full table * @return filtered table containing only duplicates */ public static Hashtable> filterDuplicates(Hashtable> table) { Hashtable> dupes = new Hashtable>(); Enumeration keys = table.keys(); while (keys.hasMoreElements()) { String hash = keys.nextElement(); List files = table.get(hash); if (files != null && files.size() > 1) { dupes.put(hash, files); } } return dupes; } /** * Convenience: find duplicates directly. */ public static Hashtable> findDuplicates(File root, boolean recursive) { Hashtable> table = buildHashTable(root, recursive); return filterDuplicates(table); } // Callback interface for GUI progress reporting public interface ProgressCallback { void onProgress(int current, int total, String file); boolean isCancelled(); } /** * Build hashtable with progress callback (for GUI). */ public static Hashtable> buildHashTableWithProgress(File root, boolean recursive, ProgressCallback cb) { Hashtable> table = new Hashtable>(); List files = collectFiles(root, recursive); int total = files.size(); int current = 0; for (File f : files) { if (cb != null && cb.isCancelled()) { break; } current++; if (cb != null) { cb.onProgress(current, total, f.getAbsolutePath()); } try { if (!f.canRead()) { System.err.println("Warning: cannot read file: " + f.getAbsolutePath()); continue; } String md5 = computeMD5(f); List list = table.get(md5); if (list == null) { list = new ArrayList(); table.put(md5, list); } list.add(f.getAbsolutePath()); } catch (IOException ex) { System.err.println("Warning: failed to hash " + f.getAbsolutePath() + " : " + ex.getMessage()); } } return table; } public static void printDuplicates(Hashtable> dupes) { if (dupes.isEmpty()) { System.out.println("No duplicate files found."); return; } // Sort hashes for deterministic output List hashes = new ArrayList(dupes.keySet()); Collections.sort(hashes); int group = 1; int totalDuplicateFiles = 0; for (String hash : hashes) { List files = dupes.get(hash); Collections.sort(files); totalDuplicateFiles += files.size(); System.out.println("=== Duplicate Group " + group + " ==="); System.out.println("MD5: " + hash + " (" + files.size() + " files)"); for (String path : files) { File f = new File(path); System.out.println(" " + path + " (" + f.length() + " bytes)"); } System.out.println(); group++; } System.out.println("Summary: " + dupes.size() + " duplicate group(s), " + totalDuplicateFiles + " files involved."); } public static void printUsage() { System.out.println("DuplicateFinder - Find duplicate files via MD5 hashtable"); System.out.println(); System.out.println("Usage:"); System.out.println(" java DuplicateFinder [options] "); System.out.println(" java DuplicateFinder --gui"); System.out.println(); System.out.println("Options:"); System.out.println(" -r, --recursive Recurse into subdirectories (default)"); System.out.println(" --no-recursive Only check top-level directory"); System.out.println(" -h, --help Show this help"); System.out.println(" --gui Launch Swing GUI"); System.out.println(); System.out.println("Examples:"); System.out.println(" java DuplicateFinder C:\\Users\\Me\\Documents"); System.out.println(" java DuplicateFinder --no-recursive C:\\Temp"); System.out.println(" java DuplicateFinder --gui"); System.out.println(" java DuplicateFinderGUI (directly launches GUI)"); } public static void main(String[] args) { boolean recursive = true; String dirArg = null; boolean showHelp = false; boolean launchGui = false; for (String arg : args) { if (arg.equals("-h") || arg.equals("--help")) { showHelp = true; } else if (arg.equals("-r") || arg.equals("--recursive")) { recursive = true; } else if (arg.equals("--no-recursive")) { recursive = false; } else if (arg.equals("--gui")) { launchGui = true; } else if (arg.startsWith("-")) { System.err.println("Unknown option: " + arg); printUsage(); System.exit(1); } else { if (dirArg == null) { dirArg = arg; } else { System.err.println("Too many positional arguments: " + arg); printUsage(); System.exit(1); } } } if (launchGui) { // Delegate to GUI DuplicateFinderGUI.main(new String[0]); return; } if (showHelp) { printUsage(); return; } if (dirArg == null) { System.err.println("Error: no directory specified."); printUsage(); System.exit(1); } File root = new File(dirArg); if (!root.exists()) { System.err.println("Error: path does not exist: " + dirArg); System.exit(1); } System.out.println("Scanning: " + root.getAbsolutePath()); System.out.println("Recursive: " + recursive); System.out.println(); long start = System.currentTimeMillis(); Hashtable> dupes = findDuplicates(root, recursive); long elapsed = System.currentTimeMillis() - start; printDuplicates(dupes); System.out.println("Scanned in " + elapsed + " ms."); } }