Limpa dependências versionadas e reconstrói catálogo com 20 projetos

scripts/handler/main.py

@@ -35,12 +35,14 @@ from utils import (
     SAIDA_ANALISES,
 )
 
+from etl_settings import worker_count
+
 # === CONFIGURAÇÕES DE SEGURANÇA E LOTES ===
 TIMEOUT_RENDER_SECONDS = 300  # 5 minutos máximo por arquivo
 MIN_RAM_FREE_MB = 500  # Mínimo de RAM livre para iniciar worker
 BATCH_SIZE = 250  # Tamanho do lote para o YAML
 LOTES_FOLDER = os.path.join(SRC_MMPSEARCH, "lotes_yaml")  # Pasta dos lotes
-LIMIT_FILES = 300  # <--- 0 = PROCESSA TUDO | >0 = LIMITA A QUANTIDADE (EX: 50)
+LIMIT_FILES = int(os.environ.get("MMPSEARCH_LIMIT_FILES", "0"))  # <--- 0 = PROCESSA TUDO | >0 = LIMITA A QUANTIDADE (EX: 50)
 
 # === VARIÁVEL GLOBAL PARA O WORKER (CORREÇÃO MULTIPROCESSING) ===
 # Necessário para evitar erro de pickle ao passar Lock via argumentos
@@ -140,11 +142,7 @@ def check_system_dependencies():
 
 def get_cpu_safe_count():
     """Retorna contagem de CPU menos 1 para não travar o SO."""
-    try:
-        count = multiprocessing.cpu_count()
-        return max(1, count - 1)
-    except:
-        return 1
+    return worker_count()
 
 
 # --- Funções de Info de Sistema ---
@@ -296,7 +294,7 @@ def process_single_file(args):
                     stderr=subprocess.PIPE,
                     check=True,
                     timeout=60,
-                    env={"QT_QPA_PLATFORM": "offscreen", **os.environ},
+                    env={**os.environ, "QT_QPA_PLATFORM": "offscreen"},
                 )
             target_mmp_path = output_mmp_path
 
@@ -314,7 +312,7 @@ def process_single_file(args):
                 capture_output=True,
                 text=True,
                 timeout=TIMEOUT_RENDER_SECONDS,
-                env={"QT_QPA_PLATFORM": "offscreen", **os.environ},
+                env={**os.environ, "QT_QPA_PLATFORM": "offscreen"},
             )
 
             if render_process.returncode != 0:
@@ -430,7 +428,7 @@ def main_parallel():
 
     # Cria pastas (incluindo a de Lotes)
     create_folders_if_not_exist(
-        [MMPZ_FOLDER, WAV_FOLDER, METADATA_FOLDER, DATA_FOLDER, LOTES_FOLDER]
+        [MMPZ_FOLDER, WAV_FOLDER, METADATA_FOLDER, DATA_FOLDER, LOTES_FOLDER, SAIDA_ANALISES]
     )
 
     logging.info(
@@ -438,14 +436,12 @@ def main_parallel():
     )
 
     if not os.path.exists(MMP_FOLDER):
-        logging.critical(f"Pasta {MMP_FOLDER} não encontrada.")
-        return
+        raise RuntimeError(f"Pasta {MMP_FOLDER} nao encontrada.")
 
-    all_files_raw = [f for f in os.listdir(MMP_FOLDER) if f.endswith((".mmp", ".mmpz"))]
+    all_files_raw = sorted(f for f in os.listdir(MMP_FOLDER) if f.endswith((".mmp", ".mmpz")))
 
     if not all_files_raw:
-        logging.warning("Nenhum arquivo encontrado.")
-        return
+        raise RuntimeError("Nenhum projeto encontrado para processar.")
 
     # === APLICAÇÃO DO LIMITE DE ARQUIVOS (ALTERAÇÃO AQUI) ===
     if LIMIT_FILES and LIMIT_FILES > 0:
@@ -537,6 +533,9 @@ def main_parallel():
                 failed_files.append(res)
 
     # === SALVAMENTO EM LOTES (BATCHING) ===
+    if not successful_data:
+        raise RuntimeError("Nenhum projeto processado com sucesso; ETL interrompido.")
+
     if successful_data:
         logging.info(f"Total processado com sucesso: {len(successful_data)}")
         logging.info(f"Iniciando gravação em lotes (Batch Size: {BATCH_SIZE})...")
@@ -579,6 +578,9 @@ def main_parallel():
     except Exception as e:
         manifest_report = {"generated": [], "failed": [str(e)]}
 
+    if manifest_report["failed"]:
+        raise RuntimeError(f"Falha nos manifestos: {manifest_report['failed']}")
+
     duration = time.time() - start_time_global
     logging.info("=" * 60)
     logging.info(
@@ -597,8 +599,11 @@ def main_parallel():
             os.path.join(METADATA_FOLDER, "samples-manifest.json"),
             os.path.join(METADATA_FOLDER, "dependency_report.json"),
         )
-    except Exception:
-        pass
+    except Exception as error:
+        raise RuntimeError("Falha na verificacao de dependencias") from error
+
+    if failed_files and os.environ.get("MMPSEARCH_ALLOW_PARTIAL", "0") != "1":
+        raise RuntimeError(f"{len(failed_files)} projetos falharam. Revise o CSV antes de permitir resultados parciais.")
 
 
 if __name__ == "__main__":

scripts/handler/utils.py

@@ -1,21 +1,26 @@
 # utils.py
 
 import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from etl_settings import REPO_ROOT, DATA_ROOT
 
 BASE_PATH = "/var/www/html/trens"
 BASE_PATH_MMPSEARCH = "/var/www/html/trens/mmpSearch"
-BASE_DATA = "/nethome/jotachina/projetos/mmpSearch"
-SRC_MMPSEARCH = "/var/www/html/trens/src_mmpSearch"
+BASE_DATA = str(REPO_ROOT)
+SRC_MMPSEARCH = str(DATA_ROOT)
 
 SAIDA_ANALISES = os.path.join(SRC_MMPSEARCH, "saida_analises")
-MMP_FOLDER = os.path.join(BASE_PATH, "src_mmpSearch", "mmp")
-MMPZ_FOLDER = os.path.join(BASE_PATH, "src_mmpSearch", "mmpz")
-WAV_FOLDER = os.path.join(BASE_PATH, "src_mmpSearch", "wav")
-METADATA_FOLDER = os.path.join(BASE_PATH, "src_mmpSearch", "metadata")
-SAMPLE_MANIFEST = os.path.join(BASE_PATH, "src_mmpSearch", "metadata", "samples-manifest.json")
+MMP_FOLDER = os.path.join(SRC_MMPSEARCH, "mmp")
+MMPZ_FOLDER = os.path.join(SRC_MMPSEARCH, "mmpz")
+WAV_FOLDER = os.path.join(SRC_MMPSEARCH, "wav")
+METADATA_FOLDER = os.path.join(SRC_MMPSEARCH, "metadata")
+SAMPLE_MANIFEST = os.path.join(SRC_MMPSEARCH, "metadata", "samples-manifest.json")
 SAMPLE_SRC = os.path.join(SRC_MMPSEARCH, "samples")
-MMP_MANIFEST = os.path.join(BASE_PATH, "src_mmpSearch", "metadata", "mmp-manifest.json")
-DATA_FOLDER = os.path.join(BASE_DATA, "_data")
+MMP_MANIFEST = os.path.join(SRC_MMPSEARCH, "metadata", "mmp-manifest.json")
+DATA_FOLDER = os.environ.get("MMPSEARCH_SITE_DATA_ROOT", os.path.join(BASE_DATA, "_data"))
 XML_IMPORTED_PATH_PREFIX = os.path.join(SAMPLE_SRC, "imported")
 SAMPLE_MANIFEST_DATA = os.path.join(DATA_FOLDER, "samples-manifest.json")
 LOG_FOLDER = os.path.join(SRC_MMPSEARCH, "logs", "handler_logs")
@@ -44,7 +49,7 @@ CONFIGS = [
         "source_dir": WAV_FOLDER,  # Vai ler a pasta src_mmpSearch/wav
         "output_file": os.path.join(DATA_FOLDER, "beats.yml"), # Salva em _data/beats.json
         "scan_type": "list",
-        "extensions": [".wav", ".mp3"], # Extensões permitidas
+        "extensions": [".wav", ".mp3", ".ogg"], # Extensões permitidas
     },
 ]