FindDuplicateFiles/DuplicateFinder.java

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.List;

/**
 * DuplicateFinder - Command line program to find duplicate files using MD5 hashing.
 *
 * Uses a Hashtable where MD5 hash (String) -> List of file paths (String).
 * Files sharing the same MD5 are duplicates (identical content).
 *
 * Usage:
 *   java DuplicateFinder [options] <directory>
 *   java DuplicateFinder --gui   (launches Swing GUI)
 *
 * Options:
 *   -r, --recursive    Recurse into subdirectories (default: true)
 *   --no-recursive     Do not recurse
 *   -h, --help         Show help
 *   --gui              Launch Swing GUI
 *
 * Standalone CLI - can be used without GUI.
 * GUI wrapper is in DuplicateFinderGUI.java
 */
public class DuplicateFinder {

    private static final int BUFFER_SIZE = 8192;

    /**
     * Compute MD5 hash for a file and return as hex string.
     * @param file file to hash
     * @return hex MD5 string
     * @throws IOException if read fails
     */
    public static String computeMD5(File file) throws IOException {
        MessageDigest md;
        try {
            md = MessageDigest.getInstance("MD5");
        } catch (NoSuchAlgorithmException e) {
            throw new RuntimeException("MD5 not available", e);
        }

        InputStream is = null;
        try {
            is = new FileInputStream(file);
            byte[] buffer = new byte[BUFFER_SIZE];
            int read;
            while ((read = is.read(buffer)) != -1) {
                md.update(buffer, 0, read);
            }
        } finally {
            if (is != null) {
                try { is.close(); } catch (IOException ignored) {}
            }
        }

        byte[] digest = md.digest();
        StringBuilder sb = new StringBuilder(32);
        for (byte b : digest) {
            sb.append(String.format("%02x", b & 0xff));
        }
        return sb.toString();
    }

    /**
     * Collect all files under root.
     * @param root starting directory or file
     * @param recursive if true, walk subdirectories
     * @return list of files (not directories)
     */
    public static List<File> collectFiles(File root, boolean recursive) {
        List<File> files = new ArrayList<File>();
        if (root == null || !root.exists()) {
            return files;
        }
        if (root.isFile()) {
            files.add(root);
            return files;
        }
        File[] children = root.listFiles();
        if (children == null) {
            System.err.println("Warning: cannot list directory: " + root.getAbsolutePath());
            return files;
        }
        for (File child : children) {
            if (child.isFile()) {
                files.add(child);
            } else if (child.isDirectory() && recursive) {
                // avoid following symlinks that cause cycles? isFile/isDirectory follows links but
                // we simply recurse; deep cycles unlikely on Windows
                files.addAll(collectFiles(child, true));
            } else if (child.isDirectory() && !recursive) {
                // skip
            }
        }
        return files;
    }

    /**
     * Build Hashtable mapping MD5 hash -> list of file paths.
     * This is the core hashtable required by the specification.
     *
     * @param root directory to search
     * @param recursive recurse flag
     * @return Hashtable where key = md5 hex, value = list of absolute paths
     */
    public static Hashtable<String, List<String>> buildHashTable(File root, boolean recursive) {
        Hashtable<String, List<String>> table = new Hashtable<String, List<String>>();
        List<File> files = collectFiles(root, recursive);

        for (File f : files) {
            try {
                // skip zero-length? No, include - empty files duplicates too
                // skip unreadable
                if (!f.canRead()) {
                    System.err.println("Warning: cannot read file: " + f.getAbsolutePath());
                    continue;
                }
                String md5 = computeMD5(f);
                List<String> list = table.get(md5);
                if (list == null) {
                    list = new ArrayList<String>();
                    table.put(md5, list);
                }
                list.add(f.getAbsolutePath());
            } catch (IOException ex) {
                System.err.println("Warning: failed to hash " + f.getAbsolutePath() + " : " + ex.getMessage());
            }
        }
        return table;
    }

    /**
     * Filter hashtable to only entries with duplicates (more than one file per hash).
     * @param table full table
     * @return filtered table containing only duplicates
     */
    public static Hashtable<String, List<String>> filterDuplicates(Hashtable<String, List<String>> table) {
        Hashtable<String, List<String>> dupes = new Hashtable<String, List<String>>();
        Enumeration<String> keys = table.keys();
        while (keys.hasMoreElements()) {
            String hash = keys.nextElement();
            List<String> files = table.get(hash);
            if (files != null && files.size() > 1) {
                dupes.put(hash, files);
            }
        }
        return dupes;
    }

    /**
     * Convenience: find duplicates directly.
     */
    public static Hashtable<String, List<String>> findDuplicates(File root, boolean recursive) {
        Hashtable<String, List<String>> table = buildHashTable(root, recursive);
        return filterDuplicates(table);
    }

    // Callback interface for GUI progress reporting
    public interface ProgressCallback {
        void onProgress(int current, int total, String file);
        boolean isCancelled();
    }

    /**
     * Build hashtable with progress callback (for GUI).
     */
    public static Hashtable<String, List<String>> buildHashTableWithProgress(File root, boolean recursive, ProgressCallback cb) {
        Hashtable<String, List<String>> table = new Hashtable<String, List<String>>();
        List<File> files = collectFiles(root, recursive);
        int total = files.size();
        int current = 0;
        for (File f : files) {
            if (cb != null && cb.isCancelled()) {
                break;
            }
            current++;
            if (cb != null) {
                cb.onProgress(current, total, f.getAbsolutePath());
            }
            try {
                if (!f.canRead()) {
                    System.err.println("Warning: cannot read file: " + f.getAbsolutePath());
                    continue;
                }
                String md5 = computeMD5(f);
                List<String> list = table.get(md5);
                if (list == null) {
                    list = new ArrayList<String>();
                    table.put(md5, list);
                }
                list.add(f.getAbsolutePath());
            } catch (IOException ex) {
                System.err.println("Warning: failed to hash " + f.getAbsolutePath() + " : " + ex.getMessage());
            }
        }
        return table;
    }

    public static void printDuplicates(Hashtable<String, List<String>> dupes) {
        if (dupes.isEmpty()) {
            System.out.println("No duplicate files found.");
            return;
        }
        // Sort hashes for deterministic output
        List<String> hashes = new ArrayList<String>(dupes.keySet());
        Collections.sort(hashes);

        int group = 1;
        int totalDuplicateFiles = 0;
        for (String hash : hashes) {
            List<String> files = dupes.get(hash);
            Collections.sort(files);
            totalDuplicateFiles += files.size();
            System.out.println("=== Duplicate Group " + group + " ===");
            System.out.println("MD5: " + hash + "  (" + files.size() + " files)");
            for (String path : files) {
                File f = new File(path);
                System.out.println("  " + path + "  (" + f.length() + " bytes)");
            }
            System.out.println();
            group++;
        }
        System.out.println("Summary: " + dupes.size() + " duplicate group(s), " + totalDuplicateFiles + " files involved.");
    }

    public static void printUsage() {
        System.out.println("DuplicateFinder - Find duplicate files via MD5 hashtable");
        System.out.println();
        System.out.println("Usage:");
        System.out.println("  java DuplicateFinder [options] <directory>");
        System.out.println("  java DuplicateFinder --gui");
        System.out.println();
        System.out.println("Options:");
        System.out.println("  -r, --recursive       Recurse into subdirectories (default)");
        System.out.println("      --no-recursive    Only check top-level directory");
        System.out.println("  -h, --help            Show this help");
        System.out.println("      --gui             Launch Swing GUI");
        System.out.println();
        System.out.println("Examples:");
        System.out.println("  java DuplicateFinder C:\\Users\\Me\\Documents");
        System.out.println("  java DuplicateFinder --no-recursive C:\\Temp");
        System.out.println("  java DuplicateFinder --gui");
        System.out.println("  java DuplicateFinderGUI   (directly launches GUI)");
    }

    public static void main(String[] args) {
        boolean recursive = true;
        String dirArg = null;
        boolean showHelp = false;
        boolean launchGui = false;

        for (String arg : args) {
            if (arg.equals("-h") || arg.equals("--help")) {
                showHelp = true;
            } else if (arg.equals("-r") || arg.equals("--recursive")) {
                recursive = true;
            } else if (arg.equals("--no-recursive")) {
                recursive = false;
            } else if (arg.equals("--gui")) {
                launchGui = true;
            } else if (arg.startsWith("-")) {
                System.err.println("Unknown option: " + arg);
                printUsage();
                System.exit(1);
            } else {
                if (dirArg == null) {
                    dirArg = arg;
                } else {
                    System.err.println("Too many positional arguments: " + arg);
                    printUsage();
                    System.exit(1);
                }
            }
        }

        if (launchGui) {
            // Delegate to GUI
            DuplicateFinderGUI.main(new String[0]);
            return;
        }

        if (showHelp) {
            printUsage();
            return;
        }

        if (dirArg == null) {
            System.err.println("Error: no directory specified.");
            printUsage();
            System.exit(1);
        }

        File root = new File(dirArg);
        if (!root.exists()) {
            System.err.println("Error: path does not exist: " + dirArg);
            System.exit(1);
        }

        System.out.println("Scanning: " + root.getAbsolutePath());
        System.out.println("Recursive: " + recursive);
        System.out.println();

        long start = System.currentTimeMillis();

        Hashtable<String, List<String>> dupes = findDuplicates(root, recursive);

        long elapsed = System.currentTimeMillis() - start;

        printDuplicates(dupes);

        System.out.println("Scanned in " + elapsed + " ms.");
    }
}