Refactor code structure for improved readability and maintainability

picopy/copy_engine.py

@@ -0,0 +1,415 @@
+"""PiCopy – Kopierlogik: do_copy, check_auto_copy, usb_monitor."""
+
+import hashlib as _hashlib
+import os
+import re
+import shutil
+import subprocess
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+from picopy.config import load_cfg, _fmt_bytes, log
+from picopy.state import (
+    copy_state, copy_lock, save_state, append_history, add_log
+)
+from picopy.usb import usb_devices, ensure_mount, internal_dest_device
+
+_copy_thread: threading.Thread | None = None
+
+SYSTEM_EXCLUDES = {
+    '.DS_Store', 'Thumbs.db', 'thumbs.db', 'desktop.ini',
+    '.Spotlight-V100', '.Trashes', '.fseventsd', '.TemporaryItems',
+    '.VolumeIcon.icns', 'RECYCLER', '$RECYCLE.BIN',
+    'System Volume Information', '.DocumentRevisions-V100',
+}
+
+
+def _should_copy(f: Path, cfg: dict) -> bool:
+    if cfg.get('exclude_system'):
+        for part in f.parts:
+            if part in SYSTEM_EXCLUDES:
+                return False
+        if f.name.startswith('._'):
+            return False
+    filt = cfg.get('file_filter', '').strip()
+    if filt:
+        allowed = {e.strip().lower().lstrip('.') for e in filt.split(',') if e.strip()}
+        if f.suffix.lower().lstrip('.') not in allowed:
+            return False
+    return True
+
+
+def _unique_path(p: Path) -> Path:
+    stem, suffix, parent = p.stem, p.suffix, p.parent
+    i = 1
+    while True:
+        candidate = parent / f'{stem}_({i}){suffix}'
+        if not candidate.exists():
+            return candidate
+        i += 1
+
+
+def _file_md5(p: Path) -> str:
+    h = _hashlib.md5()
+    with open(p, 'rb') as f:
+        for chunk in iter(lambda: f.read(65536), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _resolve_source_ports(cfg) -> list:
+    """Gibt source_ports als [{port, label}]-Liste zurück. Migriert altes source_port-Feld."""
+    ports = cfg.get('source_ports') or []
+    if not ports and cfg.get('source_port'):
+        ports = [{'port': cfg['source_port'], 'label': cfg.get('source_label', '')}]
+    return ports
+
+
+def _configured_destination(cfg, devs):
+    if cfg.get('dest_type') == 'internal':
+        return internal_dest_device(cfg)
+    return next((d for d in devs if d['usb_port'] == cfg.get('dest_port')), None)
+
+
+def do_copy(src_devs, dst_dev, cfg):
+    """Kopiert von einer oder mehreren Quellen auf ein Ziel."""
+    dst_mp = None
+    dst_owned = False
+    src_mounts = []  # [(src_dev, src_mp, src_owned)]
+    _upload_thread = None
+    _hist = {
+        'start': time.time(),
+        'ok': False, 'copied': 0, 'skipped': 0, 'errors': 0,
+        'bytes': 0, 'error_msg': '',
+    }
+    try:
+        with copy_lock:
+            copy_state.update(running=True, progress=0, error=None,
+                              done=0, total=0, logs=[], current='',
+                              bytes_total=0, bytes_done=0,
+                              start_ts=time.time(), eta_sec=None, speed_bps=0,
+                              phase='copy',
+                              space_warning=False, space_needed=0, space_free=0,
+                              last_success_file='')
+        save_state()
+        n = len(src_devs)
+        add_log(f'Kopiervorgang gestartet ({n} Quelle{"n" if n != 1 else ""})')
+
+        dst_mp, dst_owned = ensure_mount(dst_dev)
+        if not dst_mp:
+            raise RuntimeError(f'Ziel nicht mountbar: {dst_dev["device"]}')
+        add_log(f'Ziel: {dst_mp} ({dst_dev["label"]})')
+
+        ts = datetime.now()
+        date_str = ts.strftime(cfg['folder_format'])
+        if cfg.get('add_time'):
+            date_str += '_' + ts.strftime('%H%M%S')
+
+        # -- Alle Quellen mounten & Dateien sammeln -------------------------
+        # source_data: [(src_dev, src_path, files, dst_dir, incomplete_marker)]
+        source_data = []
+        total = 0
+        bytes_total = 0
+
+        for src_dev in src_devs:
+            with copy_lock:
+                cancelled = not copy_state['running']
+            if cancelled:
+                add_log('Abgebrochen')
+                return
+
+            src_mp_i, src_owned_i = ensure_mount(src_dev)
+            src_mounts.append((src_dev, src_mp_i, src_owned_i))
+            if not src_mp_i:
+                add_log(f'Quelle nicht mountbar: {src_dev["device"]} - übersprungen')
+                continue
+
+            add_log(f'Quelle: {src_mp_i} ({src_dev["label"]})')
+            src_path = Path(src_mp_i)
+            all_files = [f for f in src_path.rglob('*') if f.is_file()]
+            files = [f for f in all_files if _should_copy(f, cfg)]
+            n_filtered = len(all_files) - len(files)
+            if n_filtered:
+                add_log(f'{n_filtered} Dateien gefiltert ({src_dev["label"]})')
+
+            label = re.sub(r'[^\w\-]', '_', src_dev.get('label', 'source'))
+            dst_dir_i = Path(dst_mp) / date_str
+            if cfg.get('subfolder'):
+                dst_dir_i = dst_dir_i / label
+            dst_dir_i.mkdir(parents=True, exist_ok=True)
+            add_log(f'Zielordner: {dst_dir_i}')
+
+            for stale in dst_dir_i.rglob('*.picopy_tmp'):
+                stale.unlink(missing_ok=True)
+
+            incomplete_marker_i = dst_dir_i / '.picopy_incomplete'
+            import json as _json
+            incomplete_marker_i.write_text(_json.dumps({
+                'started': datetime.now().isoformat(),
+                'source':  src_dev.get('label', ''),
+            }))
+
+            total += len(files)
+            bytes_total += sum(f.stat().st_size for f in files)
+            source_data.append((src_dev, src_path, files, dst_dir_i, incomplete_marker_i))
+
+        with copy_lock:
+            copy_state['total'] = total
+            copy_state['bytes_total'] = bytes_total
+        add_log(f'{total} Dateien gesamt ({_fmt_bytes(bytes_total)})')
+
+        # -- Speicherplatz-Prüfung ------------------------------------------
+        try:
+            dst_free = shutil.disk_usage(dst_mp).free
+        except Exception:
+            dst_free = 0
+        if bytes_total > 0 and dst_free < bytes_total:
+            with copy_lock:
+                copy_state.update(space_warning=True,
+                                  space_needed=bytes_total,
+                                  space_free=dst_free)
+            add_log(
+                f'⚠ Nicht genug Speicherplatz! '
+                f'Benötigt: {_fmt_bytes(bytes_total)}, '
+                f'Verfügbar: {_fmt_bytes(dst_free)} – '
+                f'Quelle wird nicht gelöscht'
+            )
+        save_state()
+
+        # -- Phase 1: Kopieren (alle Quellen) --------------------------------
+        dup_mode = cfg.get('duplicate_handling', 'skip')
+        all_copied_pairs = []
+        skipped = 0
+        io_errors = 0
+        global_done = 0
+
+        for src_dev_i, src_path_i, files_i, dst_dir_i, _ in source_data:
+            if len(src_devs) > 1:
+                add_log(f'Kopiere: {src_dev_i["label"]}')
+            for f in files_i:
+                with copy_lock:
+                    cancelled = not copy_state['running']
+                if cancelled:
+                    add_log('Abgebrochen')
+                    return
+                global_done += 1
+                rel   = f.relative_to(src_path_i)
+                dst_f = dst_dir_i / rel
+                try:
+                    dst_f.parent.mkdir(parents=True, exist_ok=True)
+                except OSError as mkdir_err:
+                    io_errors += 1
+                    add_log(f'⚠ Verzeichnis nicht erstellbar ({dst_f.parent.name}): {mkdir_err}')
+                    with copy_lock:
+                        copy_state.update(done=global_done,
+                                          progress=int(global_done/total*100) if total else 100,
+                                          current=str(f.name))
+                    continue
+
+                if dst_f.exists():
+                    if dup_mode == 'skip':
+                        if dst_f.stat().st_size == f.stat().st_size:
+                            skipped += 1
+                            with copy_lock:
+                                copy_state.update(done=global_done,
+                                                  progress=int(global_done/total*100) if total else 100,
+                                                  current=str(f.name))
+                            continue
+                        else:
+                            add_log(f'Unvollständige Datei, wird neu kopiert: {f.name}')
+                    elif dup_mode == 'rename':
+                        dst_f = _unique_path(dst_f)
+
+                fsize = f.stat().st_size
+                tmp_f = dst_f.with_name(dst_f.name + '.picopy_tmp')
+                try:
+                    shutil.copy2(f, tmp_f)
+                    os.replace(str(tmp_f), str(dst_f))
+                except OSError as copy_err:
+                    try: tmp_f.unlink(missing_ok=True)
+                    except Exception: pass
+                    io_errors += 1
+                    add_log(f'⚠ Fehler bei {f.name}: {copy_err}')
+                    with copy_lock:
+                        copy_state.update(done=global_done,
+                                          progress=int(global_done/total*100) if total else 100,
+                                          current=str(f.name))
+                    continue
+                all_copied_pairs.append((f, dst_f))
+
+                with copy_lock:
+                    copy_state['bytes_done'] += fsize
+                    copy_state['last_success_file'] = str(dst_f)
+                    bd = copy_state['bytes_done']
+                    bt = copy_state['bytes_total']
+                    elapsed = time.time() - copy_state['start_ts']
+                    speed = bd / elapsed if elapsed > 1 else 0
+                    eta = int((bt - bd) / speed) if speed > 0 and bt > bd else 0
+                    copy_state.update(done=global_done,
+                                      progress=int(global_done/total*100) if total else 100,
+                                      current=str(f.name), speed_bps=int(speed), eta_sec=eta)
+                if global_done % 20 == 0:
+                    save_state()
+
+        msg_parts = [f'{len(all_copied_pairs)} kopiert']
+        if skipped:
+            msg_parts.append(f'{skipped} übersprungen')
+        if io_errors:
+            msg_parts.append(f'{io_errors} Fehler (I/O)')
+
+        # -- Phase 2: Verifizieren ------------------------------------------
+        verify_errors = 0
+        verified_pairs = list(all_copied_pairs)
+
+        if cfg.get('verify_checksum') and all_copied_pairs:
+            with copy_lock:
+                copy_state.update(phase='verify', progress=0, done=0,
+                                  total=len(all_copied_pairs), current='',
+                                  eta_sec=None, speed_bps=0)
+            add_log(f'Verifiziere {len(all_copied_pairs)} Dateien...')
+            verified_pairs = []
+
+            for i, (src_f, dst_f) in enumerate(all_copied_pairs):
+                with copy_lock:
+                    cancelled = not copy_state['running']
+                    if not cancelled:
+                        copy_state.update(done=i+1,
+                                          progress=int((i+1)/len(all_copied_pairs)*100),
+                                          current=src_f.name)
+                if cancelled:
+                    add_log('Abgebrochen')
+                    return
+                if _file_md5(src_f) == _file_md5(dst_f):
+                    verified_pairs.append((src_f, dst_f))
+                else:
+                    verify_errors += 1
+                    add_log(f'⚠ Prüfsummenfehler: {src_f.name}')
+                    try: dst_f.unlink()
+                    except Exception: pass
+
+            if verify_errors:
+                msg_parts.append(f'{verify_errors} Prüfsummenfehler!')
+                add_log(f'Verifizierung: {verify_errors} Fehler!')
+            else:
+                add_log(f'Alle {len(verified_pairs)} Dateien verifiziert ✓')
+
+        # -- Phase 3: Quelle löschen ----------------------------------------
+        if cfg.get('delete_source') and verified_pairs:
+            with copy_lock:
+                _space_warn = copy_state.get('space_warning', False)
+            if _space_warn:
+                add_log('Quelldateien NICHT gelöscht (Speicherplatz unzureichend)')
+            elif verify_errors:
+                add_log('Quelldateien NICHT gelöscht (Prüfsummenfehler)')
+            else:
+                with copy_lock:
+                    copy_state.update(phase='delete', current='')
+                add_log(f'Lösche {len(verified_pairs)} Quelldateien...')
+                del_errors = 0
+                for src_f, _ in verified_pairs:
+                    try:
+                        src_f.unlink()
+                    except Exception as e:
+                        del_errors += 1
+                        log.warning(f'Löschen fehlgeschlagen: {src_f}: {e}')
+                if del_errors:
+                    msg_parts.append(f'{del_errors} Löschfehler')
+                else:
+                    add_log('Quelle geleert ✓')
+
+        subprocess.run(['sync'], capture_output=True)
+        for _, _, _, _, incomplete_marker_i in source_data:
+            try: incomplete_marker_i.unlink(missing_ok=True)
+            except Exception: pass
+
+        with copy_lock:
+            copy_state['last_copy'] = datetime.now().isoformat()
+            _hist['bytes'] = copy_state['bytes_done']
+        _hist.update(ok=True, copied=len(all_copied_pairs),
+                     skipped=skipped, errors=io_errors)
+        add_log('Fertig! ' + ', '.join(msg_parts))
+
+        dst_dir_root = Path(dst_mp) / date_str
+        upload_files = [dst_f for _, dst_f in verified_pairs if dst_f.exists()]
+        if upload_files:
+            from picopy.upload import run_uploads
+            _upload_thread = threading.Thread(
+                target=run_uploads,
+                args=(dst_dir_root, cfg, upload_files),
+                daemon=True
+            )
+            _upload_thread.start()
+        elif any(t.get('enabled') for t in cfg.get('upload_targets', [])):
+            add_log('NAS-Upload: keine neu auf das Ziel übertragenen Dateien')
+
+    except Exception as e:
+        log.exception('Copy failed')
+        with copy_lock:
+            copy_state['error'] = str(e)
+        _hist['error_msg'] = str(e)
+        add_log(f'Fehler: {e}')
+
+    finally:
+        # Erst warten bis NAS-Upload fertig, dann erst unmounten
+        if _upload_thread is not None and _upload_thread.is_alive():
+            add_log('Warte auf NAS-Upload vor Unmount...')
+            _upload_thread.join()
+        subprocess.run(['sync'], capture_output=True)
+        for _, src_mp_i, src_owned_i in src_mounts:
+            if src_owned_i and src_mp_i:
+                subprocess.run(['umount', src_mp_i], capture_output=True)
+        if dst_owned and dst_mp:
+            subprocess.run(['umount', dst_mp], capture_output=True)
+        with copy_lock:
+            copy_state['running'] = False
+            copy_state['current'] = ''
+            copy_state['phase'] = 'idle'
+        save_state()
+        # Verlaufseintrag speichern
+        append_history({
+            'ts':       datetime.now().isoformat(),
+            'duration': int(time.time() - _hist['start']),
+            'sources':  [d.get('label', d.get('device', '?')) for d in src_devs],
+            'dest':     dst_dev.get('label', dst_dev.get('device', '?')) if dst_dev else '?',
+            'copied':   _hist['copied'],
+            'skipped':  _hist['skipped'],
+            'errors':   _hist['errors'],
+            'bytes':    _hist['bytes'],
+            'ok':       _hist['ok'],
+            'error':    _hist['error_msg'],
+        })
+
+
+def check_auto_copy():
+    cfg = load_cfg()
+    src_ports = _resolve_source_ports(cfg)
+    if not cfg.get('auto_copy') or not src_ports:
+        return
+    if cfg.get('dest_type') != 'internal' and not cfg.get('dest_port'):
+        return
+    with copy_lock:
+        if copy_state['running'] or copy_state['error']:
+            return
+    devs = usb_devices()
+    srcs = [next((d for d in devs if d['usb_port'] == sp['port']), None) for sp in src_ports]
+    srcs = [s for s in srcs if s is not None]
+    dst  = _configured_destination(cfg, devs)
+    if srcs and dst:
+        log.info(f'Auto-Copy: {len(srcs)} Quelle(n) und Ziel verbunden')
+        threading.Thread(target=do_copy, args=(srcs, dst, cfg), daemon=True).start()
+
+
+def usb_monitor():
+    try:
+        import pyudev
+        ctx = pyudev.Context()
+        mon = pyudev.Monitor.from_netlink(ctx)
+        mon.filter_by(subsystem='block', device_type='disk')
+        for dev in iter(mon.poll, None):
+            if dev.action == 'add':
+                log.info(f'USB eingesteckt: {dev.device_node}')
+                threading.Timer(3.0, check_auto_copy).start()
+    except ImportError:
+        log.warning('pyudev nicht verfügbar')