web_downloader.py

import os
import threading
import re
import json
from flask import Flask, render_template, request, jsonify, send_file
from yt_dlp import YoutubeDL
import tempfile
import zipfile
from datetime import datetime
# Add mutagen import for MP3 tagging
try:
    from mutagen.easyid3 import EasyID3
    from mutagen.id3 import ID3, APIC, error as ID3Error, TIT2, TPE1, TALB, TDRC, TCON, COMM, TRCK
except ImportError:
    EasyID3 = None  # Will check at runtime
import requests

app = Flask(__name__)

# Global variable to store download status
download_status = {
    'is_downloading': False,
    'progress': 0,
    'message': '',
    'files': [],
    'url': '',
    'download_dir': ''
}

def detect_url_type(url):
    """Detect if URL is YouTube, YouTube Music, or Spaces"""
    url_lower = url.lower()
    if 'youtube.com' in url_lower or 'youtu.be' in url_lower:
        if 'music.youtube.com' in url_lower:
            return 'youtube_music'
        else:
            return 'youtube'
    elif 'x.com' in url_lower or 'twitter.com' in url_lower:
        return 'spaces'
    else:
        return 'unknown'

def sanitize_filename(filename):
    """Remove invalid characters from filename"""
    # Remove or replace invalid characters
    invalid_chars = '<>:"/\\|?*'
    for char in invalid_chars:
        filename = filename.replace(char, '_')
    # Remove leading/trailing spaces and dots
    filename = filename.strip(' .')
    # Limit length
    if len(filename) > 100:
        filename = filename[:100]
    return filename

def extract_metadata(url):
    """Extract metadata from URL to use for directory naming"""
    try:
        ydl_opts = {
            'quiet': True,
            'no_warnings': True,
            'extract_flat': True,
        }
        
        with YoutubeDL(ydl_opts) as ydl:
            info = ydl.extract_info(url, download=False)
            
            if info.get('_type') == 'playlist':
                # Playlist
                playlist_title = info.get('title', 'Unknown Playlist')
                return sanitize_filename(playlist_title)
            else:
                # Single video/audio
                title = info.get('title', 'Unknown Title')
                uploader = info.get('uploader', 'Unknown Uploader')
                # Combine uploader and title for better organization
                return sanitize_filename(f"{uploader} - {title}")
                
    except Exception as e:
        print(f"Error extracting metadata: {e}")
        return None

def load_playlist_index(download_dir):
    """Load playlist index from file"""
    index_path = os.path.join(download_dir, 'playlist_index.json')
    if os.path.exists(index_path):
        try:
            with open(index_path, 'r', encoding='utf-8') as f:
                return json.load(f)
        except Exception as e:
            print(f"Error loading playlist index: {e}")
    return {
        'url': '',
        'downloaded_items': [],
        'total_items': 0,
        'last_updated': ''
    }

def save_playlist_index(download_dir, index_data):
    """Save playlist index to file"""
    index_path = os.path.join(download_dir, 'playlist_index.json')
    try:
        with open(index_path, 'w', encoding='utf-8') as f:
            json.dump(index_data, f, indent=2, ensure_ascii=False)
    except Exception as e:
        print(f"Error saving playlist index: {e}")

def update_playlist_index(download_dir, url, item_info):
    """Update playlist index with new downloaded item"""
    index_data = load_playlist_index(download_dir)
    index_data['url'] = url
    index_data['last_updated'] = datetime.now().isoformat()
    
    # Add item if not already in the list
    item_id = item_info.get('id', '')
    if item_id and not any(item['id'] == item_id for item in index_data['downloaded_items']):
        index_data['downloaded_items'].append({
            'id': item_id,
            'title': item_info.get('title', 'Unknown'),
            'uploader': item_info.get('uploader', 'Unknown'),
            'duration': item_info.get('duration', 0),
            'filename': item_info.get('filename', ''),
            'downloaded_at': datetime.now().isoformat()
        })
    
    save_playlist_index(download_dir, index_data)

def create_readme_file(download_dir, url, format_type, download_playlist, playlist_limit=None):
    """Create a README.md file with download information"""
    readme_path = os.path.join(download_dir, 'README.md')
    
    # Load playlist index if it exists
    index_data = load_playlist_index(download_dir)
    
    content = f"""# Media Download

## Download Information
- **URL**: {url}
- **Format**: {format_type.title()}
- **Playlist**: {'Yes' if download_playlist else 'No'}
"""
    
    if download_playlist and playlist_limit:
        content += f"- **Playlist Limit**: {playlist_limit} items\n"
    
    content += f"- **Download Date**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n## Playlist Index\n"
    
    if index_data['downloaded_items']:
        content += f"- **Total Downloaded**: {len(index_data['downloaded_items'])} items\n"
        content += f"- **Last Updated**: {index_data['last_updated']}\n\n"
        content += "### Downloaded Items:\n"
        for i, item in enumerate(index_data['downloaded_items'], 1):
            duration_str = f" ({item['duration']}s)" if item['duration'] else ""
            content += f"{i}. **{item['title']}** by {item['uploader']}{duration_str}\n"
    else:
        content += "- No playlist items downloaded yet\n"
    
    content += "\n## Files\n"
    
    # Add list of downloaded files
    if os.path.exists(download_dir):
        for file in os.listdir(download_dir):
            if file not in ['README.md', 'playlist_index.json'] and os.path.isfile(os.path.join(download_dir, file)):
                content += f"- {file}\n"
    
    try:
        with open(readme_path, 'w', encoding='utf-8') as f:
            f.write(content)
    except Exception as e:
        print(f"Error creating README file: {e}")

def write_mp3_metadata(mp3_path, info_dict):
    """Write enriched metadata tags and embed cover art to an MP3 file using info_dict from yt-dlp."""
    if EasyID3 is None:
        print("mutagen not installed, skipping MP3 tagging.")
        return
    try:
        audio = EasyID3(mp3_path)
    except Exception:
        audio = EasyID3()
    # Set all relevant tags
    if 'title' in info_dict:
        audio['title'] = info_dict['title']
    if 'artist' in info_dict:
        audio['artist'] = info_dict['artist']
    elif 'uploader' in info_dict:
        audio['artist'] = info_dict['uploader']
    if 'album' in info_dict:
        audio['album'] = info_dict['album']
    if 'track_number' in info_dict:
        audio['tracknumber'] = str(info_dict['track_number'])
    elif 'track' in info_dict:
        audio['tracknumber'] = str(info_dict['track'])
    if 'release_year' in info_dict:
        audio['date'] = str(info_dict['release_year'])
    elif 'upload_date' in info_dict:
        audio['date'] = str(info_dict['upload_date'])
    if 'genre' in info_dict:
        audio['genre'] = info_dict['genre']
    if 'description' in info_dict:
        audio['comment'] = info_dict['description']
    elif 'comment' in info_dict:
        audio['comment'] = info_dict['comment']
    # Save basic tags
    try:
        audio.save(mp3_path)
    except Exception as e:
        print(f"Error saving MP3 metadata: {e}")
    # Embed cover art if available
    try:
        if 'thumbnail' in info_dict and info_dict['thumbnail']:
            img_url = info_dict['thumbnail']
            img_data = requests.get(img_url, timeout=10).content
            audio_id3 = ID3(mp3_path)
            audio_id3.add(APIC(
                encoding=3, # UTF-8
                mime='image/jpeg',
                type=3, # Cover (front)
                desc='Cover',
                data=img_data
            ))
            audio_id3.save(mp3_path)
    except Exception as e:
        print(f"Error embedding cover art: {e}")

def progress_hook(d):
    global download_status
    if d['status'] == 'downloading':
        try:
            # Get total bytes (try different possible keys)
            total = (d.get('total_bytes') or 
                    d.get('total_bytes_estimate') or 
                    d.get('filesize') or 
                    d.get('filesize_approx', 0))
            
            downloaded = d.get('downloaded_bytes', 0)
            
            if total and total > 0:
                progress = min((downloaded / total) * 100, 100)
                download_status['progress'] = progress
                
                # Get speed and ETA if available
                speed = d.get('speed')
                eta = d.get('eta')
                speed_str = f" ({speed/1024/1024:.1f} MB/s)" if speed else ""
                eta_str = f" - ETA: {eta}s" if eta else ""
                
                download_status['message'] = f"Downloading: {progress:.1f}%{speed_str}{eta_str}"
            else:
                # If we can't calculate progress, just show the status
                download_status['message'] = f"Downloading... {d.get('_percent_str', '')}"
                
        except Exception as e:
            download_status['message'] = f"Downloading... (progress calculation error: {str(e)})"
            
    elif d['status'] == 'finished':
        filename = d.get('filename', 'Unknown file')
        download_status['files'].append(filename)
        download_status['message'] = f"Processing: {os.path.basename(filename)}"
        
        # Write MP3 metadata if file is MP3 and info_dict is available
        if filename.lower().endswith('.mp3') and 'info_dict' in d:
            write_mp3_metadata(filename, d['info_dict'])
        
        # Update playlist index if this is a playlist download
        if download_status.get('is_playlist', False):
            try:
                # Extract item info from the download data
                item_info = {
                    'id': d.get('id', ''),
                    'title': d.get('title', 'Unknown'),
                    'uploader': d.get('uploader', 'Unknown'),
                    'duration': d.get('duration', 0),
                    'filename': filename
                }
                update_playlist_index(download_status['download_dir'], download_status['url'], item_info)
            except Exception as e:
                print(f"Error updating playlist index: {e}")
        
    elif d['status'] == 'error':
        error_msg = d.get('error', 'Unknown error')
        print(f"Download error occurred: {error_msg}")
        
        # For playlist downloads, log the error but don't stop the entire process
        if download_status.get('is_playlist', False):
            if 'skipped_items' not in download_status:
                download_status['skipped_items'] = []
            
            # Try to get the item title/ID for logging
            item_info = d.get('info_dict', {})
            item_title = item_info.get('title', 'Unknown item')
            item_id = item_info.get('id', 'Unknown ID')
            
            download_status['skipped_items'].append({
                'title': item_title,
                'id': item_id,
                'error': error_msg
            })
            
            download_status['message'] = f"⚠️ Skipped item: {item_title} (continuing with playlist...)"
            print(f"Skipped playlist item '{item_title}' due to error: {error_msg}")
        else:
            # For single downloads, show the error
            download_status['message'] = f"❌ Error: {error_msg}"

def postprocessor_hook(d):
    global download_status
    # Only act after postprocessing (e.g., after MP3 conversion)
    if d['status'] == 'finished':
        info_dict = d.get('info_dict', {})
        filename = d.get('filepath', d.get('filename', ''))
        # Write MP3 metadata (if not already done)
        if filename.lower().endswith('.mp3'):
            write_mp3_metadata(filename, info_dict)
        # Update playlist index if this is a playlist download
        if download_status.get('is_playlist', False):
            try:
                item_info = {
                    'id': info_dict.get('id', ''),
                    'title': info_dict.get('title', 'Unknown'),
                    'uploader': info_dict.get('uploader', 'Unknown'),
                    'duration': info_dict.get('duration', 0),
                    'filename': filename
                }
                update_playlist_index(download_status['download_dir'], download_status['url'], item_info)
            except Exception as e:
                print(f"Error updating playlist index (postprocessor): {e}")

def download_media(url, format_type, download_playlist, save_dir, playlist_limit=None):
    global download_status
    
    download_status['is_downloading'] = True
    download_status['progress'] = 0
    download_status['message'] = 'Starting download...'
    download_status['files'] = []
    download_status['url'] = url
    download_status['download_dir'] = save_dir
    download_status['is_playlist'] = download_playlist
    
    try:
        url_type = detect_url_type(url)
        download_status['message'] = f'Analyzing {url_type} URL...'
        
        if url_type == 'spaces':
            # Spaces configuration (X.com/Twitter)
            if format_type == "audio":
                # Audio only
                ydl_opts = {
                    "format": "bestaudio/best",
                    "outtmpl": os.path.join(save_dir, "%(title)s.%(ext)s"),
                    "noplaylist": True,
                    "hls_prefer_native": True,
                    "progress_hooks": [progress_hook],
                    "postprocessors": [
                        {
                            "key": "FFmpegExtractAudio",
                            "preferredcodec": "mp3",
                            "preferredquality": "192",
                        }
                    ],
                    "postprocessor_hooks": [postprocessor_hook],
                    "quiet": True,
                    "ignoreerrors": True,
                }
            else:
                # Video (best quality)
                ydl_opts = {
                    "format": "best[height<=1080]/best",
                    "outtmpl": os.path.join(save_dir, "%(title)s.%(ext)s"),
                    "noplaylist": True,
                    "hls_prefer_native": True,
                    "progress_hooks": [progress_hook],
                    "postprocessor_hooks": [postprocessor_hook],
                    "quiet": True,
                    "ignoreerrors": True,
                }
        else:
            # YouTube/YouTube Music configuration
            if format_type == "audio":
                # Audio only
                ydl_opts = {
                    "format": "bestaudio/best",
                    "outtmpl": os.path.join(save_dir, "%(title)s.%(ext)s"),
                    "noplaylist": not download_playlist,
                    "progress_hooks": [progress_hook],
                    "postprocessors": [
                        {
                            "key": "FFmpegExtractAudio",
                            "preferredcodec": "mp3",
                            "preferredquality": "192",
                        }
                    ],
                    "postprocessor_hooks": [postprocessor_hook],
                    "quiet": True,
                    "ignoreerrors": True,
                }
            else:
                # Video (best quality)
                ydl_opts = {
                    "format": "best[height<=1080]/best",
                    "outtmpl": os.path.join(save_dir, "%(title)s.%(ext)s"),
                    "noplaylist": not download_playlist,
                    "progress_hooks": [progress_hook],
                    "postprocessor_hooks": [postprocessor_hook],
                    "quiet": True,
                    "ignoreerrors": True,
                }
        
        # Add playlist limit if specified
        if download_playlist and playlist_limit and playlist_limit > 0:
            ydl_opts["playlist_items"] = f"1-{playlist_limit}"
        
        download_status['message'] = 'Starting download...'
        
        with YoutubeDL(ydl_opts) as ydl:
            ydl.download([url])
        
        # Create README file after download
        create_readme_file(save_dir, url, format_type, download_playlist, playlist_limit)
        
        download_status['progress'] = 100
        
        # Check if there were any skipped items and update the completion message
        skipped_count = len(download_status.get('skipped_items', []))
        if skipped_count > 0:
            download_status['message'] = f'🎉 Download complete! ({skipped_count} item(s) skipped due to errors)'
        else:
            download_status['message'] = '🎉 Download complete!'
        
    except Exception as e:
        download_status['message'] = f'❌ Error: {str(e)}'
        download_status['progress'] = 0
    finally:
        download_status['is_downloading'] = False

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/download', methods=['POST'])
def start_download():
    global download_status
    
    if download_status['is_downloading']:
        return jsonify({'error': 'Download already in progress'})
    
    data = request.get_json()
    url = data.get('url', '').strip()
    format_type = data.get('format', 'audio')
    download_playlist = data.get('playlist', False)
    playlist_limit = data.get('playlist_limit', None)
    custom_dir = data.get('custom_dir', '').strip()
    selected_directory = data.get('selected_directory', '').strip()
    
    if not url:
        return jsonify({'error': 'Please enter a URL'})
    
    url_type = detect_url_type(url)
    if url_type == 'unknown':
        return jsonify({'error': 'Please enter a valid YouTube, YouTube Music, or X.com Spaces URL'})
    
    # Validate playlist limit
    if playlist_limit is not None:
        try:
            playlist_limit = int(playlist_limit)
            if playlist_limit <= 0:
                return jsonify({'error': 'Playlist limit must be a positive number'})
        except (ValueError, TypeError):
            return jsonify({'error': 'Invalid playlist limit value'})
    
    # Determine base downloads directory
    if selected_directory:
        # Use user-selected directory (this will be a relative path from browser)
        # For now, we'll use it as a subdirectory within the default downloads folder
        downloads_dir = os.path.join(os.getcwd(), 'downloads', selected_directory)
    else:
        # Use default downloads directory
        downloads_dir = os.path.join(os.getcwd(), 'downloads')
    
    # Create downloads directory if it doesn't exist
    os.makedirs(downloads_dir, exist_ok=True)
    
    # Determine save directory
    if custom_dir:
        # Use custom directory if provided
        save_dir = os.path.join(downloads_dir, sanitize_filename(custom_dir))
    else:
        # Try to extract metadata for directory naming
        metadata_name = extract_metadata(url)
        if metadata_name:
            save_dir = os.path.join(downloads_dir, metadata_name)
        else:
            # Fallback to timestamp
            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
            save_dir = os.path.join(downloads_dir, f'download_{timestamp}')
    
    # Ensure directory name is unique
    original_save_dir = save_dir
    counter = 1
    while os.path.exists(save_dir):
        save_dir = f"{original_save_dir}_{counter}"
        counter += 1
    
    os.makedirs(save_dir, exist_ok=True)
    
    # Start download in background thread
    thread = threading.Thread(
        target=download_media,
        args=(url, format_type, download_playlist, save_dir, playlist_limit),
        daemon=True
    )
    thread.start()
    
    return jsonify({
        'message': 'Download started', 
        'save_dir': save_dir,
        'url': url
    })

@app.route('/status')
def get_status():
    return jsonify(download_status)

@app.route('/directories')
def get_directories():
    downloads_dir = os.path.join(os.getcwd(), 'downloads')
    if not os.path.exists(downloads_dir):
        return jsonify({'directories': []})
    
    directories = []
    try:
        for item in os.listdir(downloads_dir):
            item_path = os.path.join(downloads_dir, item)
            if os.path.isdir(item_path):
                directories.append({
                    'name': item,
                    'path': item_path
                })
        
        # Sort directories alphabetically
        directories.sort(key=lambda x: x['name'].lower())
        
    except Exception as e:
        return jsonify({'error': f'Error reading directories: {str(e)}'})
    
    return jsonify({'directories': directories})

@app.route('/downloads')
def list_downloads():
    downloads_dir = os.path.join(os.getcwd(), 'downloads')
    if not os.path.exists(downloads_dir):
        return jsonify({'downloads': []})
    
    downloads = []
    for item in os.listdir(downloads_dir):
        item_path = os.path.join(downloads_dir, item)
        if os.path.isdir(item_path):
            files = []
            readme_content = ""
            playlist_index = None
            
            for file in os.listdir(item_path):
                file_path = os.path.join(item_path, file)
                if os.path.isfile(file_path):
                    if file == 'README.md':
                        try:
                            with open(file_path, 'r', encoding='utf-8') as f:
                                readme_content = f.read()
                        except:
                            pass
                    elif file == 'playlist_index.json':
                        try:
                            with open(file_path, 'r', encoding='utf-8') as f:
                                playlist_index = json.load(f)
                        except:
                            pass
                    else:
                        files.append(file)
            
            # Extract URL from README if available
            url = ""
            if readme_content:
                url_match = re.search(r'\*\*URL\*\*: (.+)', readme_content)
                if url_match:
                    url = url_match.group(1)
            
            downloads.append({
                'folder': item,
                'files': files,
                'path': item_path,
                'url': url,
                'playlist_index': playlist_index
            })
    
    return jsonify({'downloads': downloads})

@app.route('/playlist-info/<folder>')
def get_playlist_info(folder):
    """Get playlist information for a specific download folder"""
    folder_path = os.path.join(os.getcwd(), 'downloads', folder)
    if not os.path.exists(folder_path):
        return jsonify({'error': 'Folder not found'}), 404
    
    # Load playlist index
    index_data = load_playlist_index(folder_path)
    
    if not index_data['url']:
        return jsonify({'error': 'No playlist information found'}), 404
    
    return jsonify({
        'url': index_data['url'],
        'downloaded_items': index_data['downloaded_items'],
        'total_downloaded': len(index_data['downloaded_items']),
        'last_updated': index_data['last_updated']
    })

@app.route('/continue-playlist/<folder>', methods=['POST'])
def continue_playlist_download(folder):
    """Continue downloading a playlist from where it left off"""
    global download_status
    
    if download_status['is_downloading']:
        return jsonify({'error': 'Download already in progress'})
    
    data = request.get_json()
    format_type = data.get('format', 'audio')
    playlist_limit = data.get('playlist_limit', None)
    
    folder_path = os.path.join(os.getcwd(), 'downloads', folder)
    if not os.path.exists(folder_path):
        return jsonify({'error': 'Folder not found'}), 404
    
    # Load playlist index
    index_data = load_playlist_index(folder_path)
    
    if not index_data['url']:
        return jsonify({'error': 'No playlist information found'}), 404
    
    url = index_data['url']
    downloaded_count = len(index_data['downloaded_items'])
    
    # Validate playlist limit
    if playlist_limit is not None:
        try:
            playlist_limit = int(playlist_limit)
            if playlist_limit <= 0:
                return jsonify({'error': 'Playlist limit must be a positive number'})
            if playlist_limit <= downloaded_count:
                return jsonify({'error': f'Playlist limit ({playlist_limit}) must be greater than already downloaded items ({downloaded_count})'})
        except (ValueError, TypeError):
            return jsonify({'error': 'Invalid playlist limit value'})
    
    # Start download in background thread
    thread = threading.Thread(
        target=download_media,
        args=(url, format_type, True, folder_path, playlist_limit),
        daemon=True
    )
    thread.start()
    
    return jsonify({
        'message': 'Playlist download continued', 
        'save_dir': folder_path,
        'url': url,
        'downloaded_count': downloaded_count
    })

@app.route('/download/<folder>/<filename>')
def download_file(folder, filename):
    file_path = os.path.join(os.getcwd(), 'downloads', folder, filename)
    if os.path.exists(file_path):
        return send_file(file_path, as_attachment=True)
    else:
        return jsonify({'error': 'File not found'}), 404

@app.route('/download/<folder>/zip')
def download_zip(folder):
    folder_path = os.path.join(os.getcwd(), 'downloads', folder)
    if not os.path.exists(folder_path):
        return jsonify({'error': 'Folder not found'}), 404
    
    try:
        # Create a temporary zip file with a clean name
        zip_filename = f'{folder}.zip'
        zip_path = os.path.join(os.getcwd(), 'downloads', zip_filename)
        
        # Create the zip file
        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
            for root, dirs, files in os.walk(folder_path):
                for file in files:
                    file_path = os.path.join(root, file)
                    # Use relative path within the zip
                    arcname = os.path.relpath(file_path, folder_path)
                    zipf.write(file_path, arcname)
        
        # Send the file and then clean up
        response = send_file(
            zip_path, 
            as_attachment=True,
            download_name=zip_filename,
            mimetype='application/zip'
        )
        
        # Add cleanup callback to remove the zip file after download
        @response.call_on_close
        def cleanup():
            try:
                if os.path.exists(zip_path):
                    os.remove(zip_path)
            except Exception as e:
                print(f"Error cleaning up zip file: {e}")
        
        return response
        
    except Exception as e:
        return jsonify({'error': f'Error creating zip file: {str(e)}'}), 500

if __name__ == '__main__':
    # Create downloads directory
    os.makedirs('downloads', exist_ok=True)
    
    print("🌐 Starting Media Downloader Web Interface...")
    print("📱 Open your browser and go to: http://localhost:5000")
    print("💾 Downloads will be saved to the 'downloads' folder")
    
    app.run(host='0.0.0.0', port=5000, debug=True)