From 33f19f905817f024a5051faf8b54e96826aa3df3 Mon Sep 17 00:00:00 2001 From: Jason Ross Date: Mon, 8 Sep 2025 00:43:19 -0500 Subject: [PATCH] WIP. Deduplication appears to be working. Need better transparency to the user about what stage the transfer is at --- .gitignore | 3 +- src/adb_manager.py | 175 ++++++++++++++++++++++ src/file_deduplication.py | 300 ++++++++++++++++++++++++++++++++++++++ src/gui/main_window.py | 62 +++++++- 4 files changed, 534 insertions(+), 6 deletions(-) create mode 100644 src/file_deduplication.py diff --git a/.gitignore b/.gitignore index a91cd38..ed3b274 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,8 @@ build .actrc bin/ test-act.yml -pkg_dist* +pkg_dist_debian +pkg* # Python cache files __pycache__/ diff --git a/src/adb_manager.py b/src/adb_manager.py index 42615c1..a5aaef8 100644 --- a/src/adb_manager.py +++ b/src/adb_manager.py @@ -18,6 +18,8 @@ import tempfile from typing import Optional import hashlib +from file_deduplication import FileDeduplicator + def get_executable_directory() -> str: """Get the directory containing the executable or script.""" if getattr(sys, 'frozen', False): @@ -238,6 +240,10 @@ class ADBManager: self.progress_callback: Optional[Callable[[int], None]] = None self.status_callback: Optional[Callable[[str], None]] = None self.current_process: Optional[subprocess.Popen] = None + self.deduplicator = FileDeduplicator( + status_callback=self._update_status, + progress_callback=self._update_progress + ) def set_progress_callback(self, callback: Callable[[int], None]): """Set callback function for progress updates.""" @@ -753,6 +759,175 @@ class ADBManager: # (Method intentionally removed - use top-level ensure_platform_tools_in_user_dir) + def pull_folder_with_dedup(self, remote_path: str, local_path: str) -> Tuple[bool, dict]: + """Pull files from Android device with duplicate detection. + + Args: + remote_path: Source path on Android device + local_path: Destination path on local machine + + Returns: + Tuple of (success, stats_dict) where stats_dict contains: + - 'transferred': number of files transferred + - 'skipped': number of duplicate files skipped + - 'bytes_saved': bytes saved by skipping duplicates + - 'total_files': total files found + """ + stats = { + 'transferred': 0, + 'skipped': 0, + 'bytes_saved': 0, + 'total_files': 0 + } + + try: + # Create local directory + os.makedirs(local_path, exist_ok=True) + + # Get list of remote files + stdout, stderr, returncode = self.run_adb_command(['shell', 'find', remote_path, '-type', 'f'], capture_output=True) + + if returncode != 0: + self._update_status(f"Failed to list remote files: {stderr}") + return False, stats + + remote_files = [line.strip() for line in stdout.splitlines() if line.strip()] + stats['total_files'] = len(remote_files) + + if not remote_files: + self._update_status("No files found in remote directory") + return True, stats + + # Build list of existing local files + local_files = [] + for root, dirs, files in os.walk(local_path): + for file_name in files: + local_files.append(os.path.join(root, file_name)) + + # Find duplicates using deduplicator + files_to_transfer, duplicate_files = self.deduplicator.find_duplicate_files( + source_files=remote_files, + target_files=local_files, + is_remote_source=True, + is_remote_target=False, + adb_command_runner=self.run_adb_command + ) + + stats['skipped'] = len(duplicate_files) + + if duplicate_files: + bytes_saved, files_saved = self.deduplicator.calculate_transfer_savings( + duplicate_files, is_remote=True, adb_command_runner=self.run_adb_command + ) + stats['bytes_saved'] = bytes_saved + self._update_status(f"Skipping {files_saved} duplicates, saving {self.deduplicator.format_bytes(bytes_saved)}") + + # Transfer non-duplicate files + if files_to_transfer: + success = self.pull_folder(remote_path, local_path) + if success: + stats['transferred'] = len(files_to_transfer) + return success, stats + else: + self._update_status("All files already exist locally - no transfer needed") + return True, stats + + except Exception as exception: + self._update_status(f"Error during deduplication check: {exception}") + return False, stats + + def push_folder_with_dedup(self, local_path: str, remote_path: str) -> Tuple[bool, dict]: + """Push files to Android device with duplicate detection. + + Args: + local_path: Source path on local machine + remote_path: Destination path on Android device + + Returns: + Tuple of (success, stats_dict) where stats_dict contains: + - 'transferred': number of files transferred + - 'skipped': number of duplicate files skipped + - 'bytes_saved': bytes saved by skipping duplicates + - 'total_files': total files found + """ + stats = { + 'transferred': 0, + 'skipped': 0, + 'bytes_saved': 0, + 'total_files': 0 + } + + try: + if not os.path.exists(local_path): + self._update_status(f"Local path does not exist: {local_path}") + return False, stats + + # Get list of local files + local_files = [] + for root, dirs, files in os.walk(local_path): + for file_name in files: + local_files.append(os.path.join(root, file_name)) + + stats['total_files'] = len(local_files) + + if not local_files: + self._update_status("No files found in local directory") + return True, stats + + # Get list of remote files + stdout, stderr, returncode = self.run_adb_command(['shell', 'find', remote_path, '-type', 'f'], capture_output=True) + + remote_files = [] + if returncode == 0 and stdout: + remote_files = [line.strip() for line in stdout.splitlines() if line.strip()] + + # Find duplicates using deduplicator + files_to_transfer, duplicate_files = self.deduplicator.find_duplicate_files( + source_files=local_files, + target_files=remote_files, + is_remote_source=False, + is_remote_target=True, + adb_command_runner=self.run_adb_command + ) + + stats['skipped'] = len(duplicate_files) + + if duplicate_files: + bytes_saved, files_saved = self.deduplicator.calculate_transfer_savings( + duplicate_files, is_remote=False + ) + stats['bytes_saved'] = bytes_saved + self._update_status(f"Skipping {files_saved} duplicates, saving {self.deduplicator.format_bytes(bytes_saved)}") + + # Transfer non-duplicate files + if files_to_transfer: + success = self.push_folder(local_path, remote_path) + if success: + stats['transferred'] = len(files_to_transfer) + return success, stats + else: + self._update_status("All files already exist remotely - no transfer needed") + return True, stats + + except Exception as exception: + self._update_status(f"Error during deduplication check: {exception}") + return False, stats + + def check_files_identical(self, local_path: str, remote_path: str, algorithm: str = 'sha256') -> bool: + """Check if local and remote files are identical by comparing hashes. + + Args: + local_path: Path to the local file + remote_path: Path to the remote file + algorithm: Hash algorithm to use + + Returns: + True if files are identical, False otherwise + """ + return self.deduplicator.check_files_identical( + local_path, remote_path, self.run_adb_command, algorithm + ) + class LinuxMTPManager: """Manages MTP operations on Linux systems.""" diff --git a/src/file_deduplication.py b/src/file_deduplication.py new file mode 100644 index 0000000..5aace7d --- /dev/null +++ b/src/file_deduplication.py @@ -0,0 +1,300 @@ +""" +File Deduplication Module +Handles hash-based file comparison and duplicate detection for file transfers. +""" + +import os +import hashlib +from typing import Optional, Dict, List, Tuple, Callable + + +class FileDeduplicator: + """Handles file hash computation and duplicate detection.""" + + def __init__(self, status_callback: Optional[Callable[[str], None]] = None, + progress_callback: Optional[Callable[[int], None]] = None): + """Initialize the deduplicator with optional callbacks. + + Args: + status_callback: Function to call with status updates + progress_callback: Function to call with progress updates (0-100) + """ + self.status_callback = status_callback + self.progress_callback = progress_callback + + def _update_status(self, message: str) -> None: + """Update status if callback is available.""" + if self.status_callback: + self.status_callback(message) + + def _update_progress(self, percentage: int) -> None: + """Update progress if callback is available.""" + if self.progress_callback: + self.progress_callback(percentage) + + def compute_local_file_hash(self, file_path: str, algorithm: str = 'sha256') -> Optional[str]: + """Compute hash of a local file. + + Args: + file_path: Path to the local file + algorithm: Hash algorithm to use ('md5', 'sha1', 'sha256') + + Returns: + Hex digest of the file hash, or None if error + """ + try: + if not os.path.isfile(file_path): + return None + + hash_obj = hashlib.new(algorithm) + with open(file_path, 'rb') as file_handle: + # Read in chunks to handle large files efficiently + for chunk in iter(lambda: file_handle.read(8192), b""): + hash_obj.update(chunk) + return hash_obj.hexdigest() + except Exception as exception: + self._update_status(f"Error computing hash for {file_path}: {exception}") + return None + + def compute_remote_file_hash(self, remote_path: str, algorithm: str = 'sha256', + adb_command_runner: Optional[Callable] = None) -> Optional[str]: + """Compute hash of a file on the Android device using ADB shell. + + Args: + remote_path: Path to the file on the Android device + algorithm: Hash algorithm to use ('md5', 'sha1', 'sha256') + adb_command_runner: Function that runs ADB commands and returns (stdout, stderr, returncode) + + Returns: + Hex digest of the file hash, or None if error + """ + if not adb_command_runner: + self._update_status("No ADB command runner provided") + return None + + try: + # Map algorithm names to Android shell commands + hash_commands = { + 'md5': 'md5sum', + 'sha1': 'sha1sum', + 'sha256': 'sha256sum' + } + + if algorithm not in hash_commands: + self._update_status(f"Unsupported hash algorithm: {algorithm}") + return None + + cmd = ['shell', hash_commands[algorithm], remote_path] + stdout, stderr, returncode = adb_command_runner(cmd, capture_output=True) + + if returncode != 0 or not stdout: + self._update_status(f"Failed to compute remote hash: {stderr}") + return None + + # Parse output: "hash_value filename" + hash_value = stdout.split()[0] if stdout else None + return hash_value.lower() if hash_value else None + + except Exception as exception: + self._update_status(f"Error computing remote hash for {remote_path}: {exception}") + return None + + def check_files_identical(self, local_path: str, remote_path: str, + adb_command_runner: Optional[Callable] = None, + algorithm: str = 'sha256') -> bool: + """Check if local and remote files are identical by comparing hashes. + + Args: + local_path: Path to the local file + remote_path: Path to the remote file + adb_command_runner: Function that runs ADB commands + algorithm: Hash algorithm to use + + Returns: + True if files are identical, False otherwise + """ + local_hash = self.compute_local_file_hash(local_path, algorithm) + remote_hash = self.compute_remote_file_hash(remote_path, algorithm, adb_command_runner) + + if local_hash is None or remote_hash is None: + return False + + return local_hash == remote_hash + + def build_local_file_hash_map(self, file_paths: List[str], + algorithm: str = 'sha256') -> Dict[str, str]: + """Build a mapping of local file paths to their hashes. + + Args: + file_paths: List of local file paths to hash + algorithm: Hash algorithm to use + + Returns: + Dictionary mapping file paths to their hash values + """ + hash_map = {} + total_files = len(file_paths) + + for index, file_path in enumerate(file_paths): + file_hash = self.compute_local_file_hash(file_path, algorithm) + + if file_hash: + hash_map[file_path] = file_hash + + # Update progress + if total_files > 0: + progress_percentage = int((index + 1) * 100 / total_files) + self._update_progress(progress_percentage) + self._update_status(f"Computing local hashes... {index + 1}/{total_files}") + + return hash_map + + def build_remote_file_hash_map(self, file_paths: List[str], + adb_command_runner: Optional[Callable] = None, + algorithm: str = 'sha256') -> Dict[str, str]: + """Build a mapping of remote file paths to their hashes. + + Args: + file_paths: List of remote file paths to hash + adb_command_runner: Function that runs ADB commands + algorithm: Hash algorithm to use + + Returns: + Dictionary mapping file paths to their hash values + """ + if not adb_command_runner: + self._update_status("No ADB command runner provided") + return {} + + hash_map = {} + total_files = len(file_paths) + + for index, file_path in enumerate(file_paths): + file_hash = self.compute_remote_file_hash(file_path, algorithm, adb_command_runner) + + if file_hash: + hash_map[file_path] = file_hash + + # Update progress + if total_files > 0: + progress_percentage = int((index + 1) * 100 / total_files) + self._update_progress(progress_percentage) + self._update_status(f"Computing remote hashes... {index + 1}/{total_files}") + + return hash_map + + def find_duplicate_files(self, source_files: List[str], target_files: List[str], + is_remote_source: bool = False, is_remote_target: bool = False, + adb_command_runner: Optional[Callable] = None, + algorithm: str = 'sha256') -> Tuple[List[str], List[str]]: + """Find files that are duplicates between source and target lists. + + Args: + source_files: List of source file paths + target_files: List of target file paths + is_remote_source: True if source files are on Android device + is_remote_target: True if target files are on Android device + adb_command_runner: Function that runs ADB commands + algorithm: Hash algorithm to use + + Returns: + Tuple of (files_to_transfer, duplicate_files) + """ + self._update_status("Building hash maps for duplicate detection...") + + # Build hash maps for both source and target + if is_remote_source: + source_hashes = self.build_remote_file_hash_map(source_files, adb_command_runner, algorithm) + else: + source_hashes = self.build_local_file_hash_map(source_files, algorithm) + + if is_remote_target: + target_hashes = self.build_remote_file_hash_map(target_files, adb_command_runner, algorithm) + else: + target_hashes = self.build_local_file_hash_map(target_files, algorithm) + + # Find duplicates by comparing hashes + target_hash_values = set(target_hashes.values()) + files_to_transfer = [] + duplicate_files = [] + + for source_file in source_files: + source_hash = source_hashes.get(source_file) + if source_hash and source_hash in target_hash_values: + duplicate_files.append(source_file) + else: + files_to_transfer.append(source_file) + + self._update_status(f"Found {len(duplicate_files)} duplicates, {len(files_to_transfer)} files to transfer") + + return files_to_transfer, duplicate_files + + def get_file_size(self, file_path: str, is_remote: bool = False, + adb_command_runner: Optional[Callable] = None) -> Optional[int]: + """Get the size of a file in bytes. + + Args: + file_path: Path to the file + is_remote: True if file is on Android device + adb_command_runner: Function that runs ADB commands + + Returns: + File size in bytes, or None if error + """ + try: + if is_remote and adb_command_runner: + cmd = ['shell', 'stat', '-c', '%s', file_path] + stdout, stderr, returncode = adb_command_runner(cmd, capture_output=True) + + if returncode == 0 and stdout.strip().isdigit(): + return int(stdout.strip()) + else: + return None + else: + if os.path.isfile(file_path): + return os.path.getsize(file_path) + else: + return None + except Exception: + return None + + def calculate_transfer_savings(self, duplicate_files: List[str], is_remote: bool = False, + adb_command_runner: Optional[Callable] = None) -> Tuple[int, int]: + """Calculate the number of bytes and files that would be saved by skipping duplicates. + + Args: + duplicate_files: List of duplicate file paths + is_remote: True if files are on Android device + adb_command_runner: Function that runs ADB commands + + Returns: + Tuple of (bytes_saved, files_saved) + """ + bytes_saved = 0 + files_saved = 0 + + for file_path in duplicate_files: + file_size = self.get_file_size(file_path, is_remote, adb_command_runner) + if file_size is not None: + bytes_saved += file_size + files_saved += 1 + + return bytes_saved, files_saved + + def format_bytes(self, bytes_value: int) -> str: + """Format bytes into human-readable string. + + Args: + bytes_value: Number of bytes + + Returns: + Formatted string (e.g., "1.5 MB") + """ + if bytes_value < 1024: + return f"{bytes_value} B" + elif bytes_value < 1024 * 1024: + return f"{bytes_value / 1024:.1f} KB" + elif bytes_value < 1024 * 1024 * 1024: + return f"{bytes_value / (1024 * 1024):.1f} MB" + else: + return f"{bytes_value / (1024 * 1024 * 1024):.1f} GB" diff --git a/src/gui/main_window.py b/src/gui/main_window.py index 8d00c88..070738f 100644 --- a/src/gui/main_window.py +++ b/src/gui/main_window.py @@ -830,14 +830,25 @@ class AndroidFileHandlerGUI(tk.Tk): filename = source_path.split("/")[-1] full_dest_path = os.path.join(dest_path, filename) success = transfer_method(source_path, full_dest_path) + stats = None # File transfers don't return stats else: - success = transfer_method(source_path, dest_path) + # For folder transfers, the method returns (success, stats) + if transfer_type == "folder": + success, stats = transfer_method(source_path, dest_path) + else: + success = transfer_method(source_path, dest_path) + stats = None if success and self.current_transfer_id == transfer_id: self._stop_transfer_animation() transfer_desc = "File" if is_file else "Folder" self._update_status(f"{transfer_desc} transfer completed successfully. To start another transfer, please select another file or folder.") self.show_disable_debugging_reminder() + + # Show transfer statistics for folder transfers + if stats is not None: + self.after(0, lambda: self._show_transfer_stats(stats, direction.capitalize())) + # Clear paths and disable button for next transfer self.after(0, self._clear_paths_and_disable_button) except Exception as e: @@ -859,11 +870,13 @@ class AndroidFileHandlerGUI(tk.Tk): if self.current_transfer_id != transfer_id: return - success = self.adb_manager.pull_folder(remote_path, local_path) + success, stats = self.adb_manager.pull_folder_with_dedup(remote_path, local_path) if success and self.current_transfer_id == transfer_id: self._stop_transfer_animation() self._update_status("Transfer completed successfully. To start another transfer, please select another file or folder.") self.show_disable_debugging_reminder() + # Show transfer statistics + self.after(0, lambda: self._show_transfer_stats(stats, "Pull")) # Clear paths and disable button for next transfer self.after(0, self._clear_paths_and_disable_button) except Exception as e: @@ -884,11 +897,13 @@ class AndroidFileHandlerGUI(tk.Tk): if self.current_transfer_id != transfer_id: return - success = self.adb_manager.push_folder(local_path, remote_path) + success, stats = self.adb_manager.push_folder_with_dedup(local_path, remote_path) if success and self.current_transfer_id == transfer_id: self._stop_transfer_animation() self._update_status("Transfer completed successfully.") self.show_disable_debugging_reminder() + # Show transfer statistics + self.after(0, lambda: self._show_transfer_stats(stats, "Push")) # Clear paths and disable button for next transfer self.after(0, self._clear_paths_and_disable_button) except Exception as e: @@ -943,12 +958,12 @@ class AndroidFileHandlerGUI(tk.Tk): if is_file: return self.adb_manager.pull_file, "file" else: - return self.adb_manager.pull_folder, "folder" + return self.adb_manager.pull_folder_with_dedup, "folder" else: # push if is_file: return self.adb_manager.push_file, "file" else: - return self.adb_manager.push_folder, "folder" + return self.adb_manager.push_folder_with_dedup, "folder" def report_error(self, message: str): """Report an error to the user (thread-safe).""" @@ -961,6 +976,43 @@ class AndroidFileHandlerGUI(tk.Tk): messagebox.showerror("Error", message) self.enable_controls() + def _show_transfer_stats(self, stats: dict, operation: str): + """Show transfer statistics dialog. + + Args: + stats: Dictionary containing transfer statistics + operation: Type of operation ("Pull" or "Push") + """ + # Format bytes saved + bytes_saved_str = "" + if stats['bytes_saved'] > 0: + # Use the deduplicator's format_bytes method + if hasattr(self.adb_manager, 'deduplicator'): + bytes_saved_str = f" ({self.adb_manager.deduplicator.format_bytes(stats['bytes_saved'])} saved)" + else: + bytes_saved_str = f" ({stats['bytes_saved']} bytes saved)" + + # Build message + title = f"{operation} Transfer Complete" + + if stats['total_files'] == 0: + message = "No files were found to transfer." + else: + message_parts = [ + f"Transfer completed successfully!\n", + f"Files found: {stats['total_files']}", + f"Files transferred: {stats['transferred']}", + f"Duplicate files skipped: {stats['skipped']}{bytes_saved_str}" + ] + + if stats['skipped'] > 0: + message_parts.append(f"\nDuplicate detection helped avoid unnecessary transfers!") + + message = "\n".join(message_parts) + + # Show dialog + messagebox.showinfo(title, message) + def on_close(self): """Handle window close event.""" self.destroy()