att base de dados

.gitignore

@@ -10,6 +10,5 @@ venv
 .gitignore
 .git
 .gitea
-scripts
 src
 assets/js/creations/server/data

scripts/classificacao/analise_audio.py

@@ -0,0 +1,631 @@
+import os
+import sys
+import multiprocessing
+import concurrent.futures  # NOVO: Necessário para usar o ProcessPoolExecutor
+import csv
+import time
+import glob
+from datetime import datetime, timedelta
+
+# ================= 1. LIMITADOR DE CPU DINÂMICO =================
+def configurar_limites_cpu():
+    try:
+        total_cores = multiprocessing.cpu_count()
+
+        print("--- Configuração de Hardware ---")
+        print(f"Núcleos Totais: {total_cores}")
+        print("Núcleos Alocados: 1 thread por processo (Multiprocessing Real)")
+        print("Núcleos Reservados para o Sistema: 1")
+
+        # IMPORTANTE: Em processamento paralelo real (ProcessPoolExecutor),
+        # as libs em C++ devem usar apenas "1" thread para evitar colisão na CPU (oversubscription)
+        os.environ["OMP_NUM_THREADS"] = "1"
+        os.environ["MKL_NUM_THREADS"] = "1"
+        os.environ["TF_NUM_INTRAOP_THREADS"] = "1"
+        os.environ["TF_NUM_INTEROP_THREADS"] = "1"
+        os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+
+    except Exception as e:
+        print(f"Aviso: Não foi possível limitar CPU automaticamente: {e}")
+
+configurar_limites_cpu()
+
+# ================= IMPORTS =================
+import json
+import yaml
+import logging
+import gc
+import numpy as np
+import psutil
+import essentia
+import essentia.standard as es
+from tqdm import tqdm
+import unicodedata
+
+# Desativa logs internos do Essentia
+essentia.log.info.active = False
+essentia.log.warning.active = False
+
+# ================= CONFIGURAÇÕES =================
+BASE_DIR = "/var/www/html/trens/src_mmpSearch"
+PASTA_WAVS = os.path.join(BASE_DIR, "wav")
+PASTA_LOTES_YAML = os.path.join(BASE_DIR, "lotes_yaml")
+
+# --- SISTEMA DE DOIS MODELOS ---
+MODELO_EMBEDDING = "discogs-effnet-bs64-1.pb"
+MODELO_CLASSIFIER = "genre_discogs400.pb"
+MODELO_CLASSES = "genre_discogs400.json"
+
+# Saídas
+DIR_SAIDA = os.path.join(BASE_DIR, "saida_analises")
+os.makedirs(DIR_SAIDA, exist_ok=True)
+ARQUIVO_CHECKPOINT = os.path.join(DIR_SAIDA, "checkpoint_analises.jsonl")
+ARQUIVO_FINAL_JSON = os.path.join(DIR_SAIDA, "db_final_completo.json")
+ARQUIVO_AUDITORIA = os.path.join(DIR_SAIDA, "audit_performance.csv")
+
+# Limites de Segurança
+LIMITE_RAM_MINIMA_MB = 500
+MAX_DURACAO_ANALISE = 20 * 60
+
+# Logs
+LOG_DIR = os.path.join(BASE_DIR, "logs/classificacao")
+os.makedirs(LOG_DIR, exist_ok=True)
+TIMESTAMP = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+logging.basicConfig(
+    filename=os.path.join(LOG_DIR, f"log_{TIMESTAMP}.log"),
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+)
+
+DATABASE_YAML = {}
+
+# Variáveis globais para os Workers conseguirem enxergar os modelos em memória
+GLOBAL_EMBEDDING = None
+GLOBAL_CLASSIFIER = None
+GLOBAL_CLASSES = None
+
+# ================= CLASSE DE AUDITORIA OTIMIZADA =================
+class Auditoria:
+    def __init__(self, arquivo_csv):
+        self.arquivo_csv = arquivo_csv
+        self.inicio_global = time.time()
+        self.process = psutil.Process(os.getpid())
+
+        # Mantém o arquivo aberto em modo 'append' na memória (Resolve Gargalo HD)
+        cabecalho = not os.path.exists(self.arquivo_csv)
+        self.file = open(self.arquivo_csv, "a", newline="", encoding="utf-8")
+        self.writer = csv.writer(self.file)
+
+        if cabecalho:
+            self.writer.writerow(
+                [
+                    "timestamp",
+                    "arquivo",
+                    "tamanho_mb",
+                    "duracao_audio_s",
+                    "tempo_proc_s",
+                    "ram_uso_mb",
+                    "cpu_percent",
+                ]
+            )
+            self.file.flush()
+
+    def registrar_processamento(self, nome_arquivo, tamanho_bytes, duracao_audio, tempo_gasto):
+        try:
+            ram_mb = self.process.memory_info().rss / (1024 * 1024)
+            cpu_pct = self.process.cpu_percent(interval=None)
+            tamanho_mb = tamanho_bytes / (1024 * 1024)
+            agora = datetime.now().strftime("%H:%M:%S")
+
+            # Escreve na mesma conexão de arquivo aberta
+            self.writer.writerow(
+                [
+                    agora,
+                    nome_arquivo,
+                    round(tamanho_mb, 2),
+                    round(duracao_audio, 2),
+                    round(tempo_gasto, 4),
+                    round(ram_mb, 2),
+                    cpu_pct,
+                ]
+            )
+            self.file.flush()
+        except Exception as e:
+            logging.error(f"Erro ao auditar arquivo: {e}")
+
+    def verificar_marco(self, contagem):
+        pass # Você pode customizar logs de progresso aqui depois se quiser
+
+    def fechar(self):
+        try:
+            self.file.close()
+        except:
+            pass
+
+
+# ================= FUNÇÕES AUXILIARES =================
+
+def verificar_memoria_segura(tamanho_arquivo_bytes):
+    mem = psutil.virtual_memory()
+    livre_mb = mem.available / (1024 * 1024)
+    estimativa_uso_ram = tamanho_arquivo_bytes * 15
+    uso_mb = estimativa_uso_ram / (1024 * 1024)
+
+    if livre_mb < LIMITE_RAM_MINIMA_MB:
+        print(f" [!] RAM Crítica ({livre_mb:.1f}MB livres). Forçando limpeza...")
+        gc.collect()
+        time.sleep(2)
+        mem = psutil.virtual_memory()
+        if mem.available / (1024 * 1024) < LIMITE_RAM_MINIMA_MB:
+            return False, "RAM insuficiente no sistema"
+
+    if uso_mb > (livre_mb * 0.8):
+        return (
+            False,
+            f"Arquivo muito grande para RAM atual (Est: {uso_mb:.1f}MB, Livre: {livre_mb:.1f}MB)",
+        )
+
+    return True, "OK"
+
+
+def normalizar_chave(texto):
+    if not texto:
+        return ""
+    s = str(texto)
+    if "/" in s:
+        s = os.path.basename(s)
+    s = os.path.splitext(s)[0]
+    s = unicodedata.normalize("NFKD", s).encode("ASCII", "ignore").decode("ASCII")
+    return s.lower().replace(" ", "").replace("-", "").replace("_", "").strip()
+
+
+def carregar_database_yaml():
+    print(f"--- Carregando Metadados de Lotes YAML em: {PASTA_LOTES_YAML} ---")
+
+    padrao = os.path.join(PASTA_LOTES_YAML, "*.yml")
+    arquivos_lote = sorted(glob.glob(padrao))
+
+    db = {}
+
+    if not arquivos_lote:
+        logging.warning(f"Nenhum lote encontrado em {PASTA_LOTES_YAML}")
+        return {}
+
+    total_carregados = 0
+
+    for arquivo in arquivos_lote:
+        try:
+            with open(arquivo, "r", encoding="utf-8") as f:
+                dados_lote = yaml.safe_load(f)
+
+            if not dados_lote:
+                continue
+
+            lista = dados_lote if isinstance(dados_lote, list) else [dados_lote]
+
+            for proj in lista:
+                if not isinstance(proj, dict):
+                    continue
+
+                candidatos = [
+                    proj.get("file"),
+                    proj.get("original_title"),
+                    proj.get("src"),
+                ]
+                for k in candidatos:
+                    if k:
+                        chave = normalizar_chave(k)
+                        if chave:
+                            db[chave] = proj
+
+            total_carregados += len(lista)
+            print(f"-> Carregado: {os.path.basename(arquivo)} ({len(lista)} projetos)")
+
+        except Exception as e:
+            logging.error(f"Erro ao ler lote {arquivo}: {e}")
+            print(f"X Erro ao ler {os.path.basename(arquivo)}")
+
+    print(f"--- Total de Projetos no DB (Memória): {len(db)} (de {total_carregados} lidos) ---")
+    return db
+
+
+def carregar_checkpoint():
+    processados = set()
+    dados_existentes = []
+    if os.path.exists(ARQUIVO_CHECKPOINT):
+        with open(ARQUIVO_CHECKPOINT, "r", encoding="utf-8") as f:
+            for linha in f:
+                try:
+                    obj = json.loads(linha)
+                    reprocessar = False
+                    if "analise_ia" in obj and "erro" in obj["analise_ia"]:
+                        reprocessar = True
+                    if "analise_tecnica" in obj:
+                        est = obj["analise_tecnica"].get("estrutura_segundos", [])
+                        if len(est) == 2 and est[1] == 12.0:
+                            reprocessar = True
+                    if reprocessar:
+                        continue
+                    processados.add(obj["arquivo"])
+                    dados_existentes.append(obj)
+                except:
+                    pass
+    return processados, dados_existentes
+
+
+def salvar_progresso(resultado):
+    with open(ARQUIVO_CHECKPOINT, "a", encoding="utf-8") as f:
+        json.dump(resultado, f, ensure_ascii=False)
+        f.write("\n")
+
+
+def calcular_complexidade(projeto_yaml):
+    score = 0
+    W_INST_SYNTH, W_INST_SAMPLE = 3.0, 1.0
+    W_AUTOMATION, W_PATTERN, W_FX = 2.0, 0.5, 1.5
+
+    stats = {"num_automations": 0, "num_patterns": 0, "num_effects": 0, "num_synths": 0}
+
+    tracks = projeto_yaml.get("tracks", [])
+    for track in tracks:
+        instruments = track.get("instruments", [])
+        for inst in instruments:
+            nome_plugin = inst.get("plugin_name", "").lower()
+            nome_inst = inst.get("instrument_name", "").lower()
+
+            if "automation" in nome_inst or "automation" in nome_plugin:
+                score += W_AUTOMATION
+                stats["num_automations"] += 1
+            elif nome_plugin in [
+                "zynaddsubfx",
+                "monstro",
+                "lb302",
+                "watsyn",
+                "opyd",
+                "sfxr",
+            ]:
+                score += W_INST_SYNTH
+                stats["num_synths"] += 1
+            else:
+                score += W_INST_SAMPLE
+
+            patterns = inst.get("patterns", [])
+            qtd_patterns = len(patterns)
+            score += qtd_patterns * W_PATTERN
+            stats["num_patterns"] += qtd_patterns
+
+            fxchain = inst.get("fxchain", {})
+            if str(fxchain.get("enabled")) == "1":
+                num_fx = int(fxchain.get("numofeffects", 0))
+                score += num_fx * W_FX
+                stats["num_effects"] += num_fx
+
+    if score <= 15:
+        estrelas = 1
+    elif score <= 40:
+        estrelas = 2
+    elif score <= 80:
+        estrelas = 3
+    elif score <= 120:
+        estrelas = 4
+    else:
+        estrelas = 5
+
+    return {"score_total": round(score, 2), "estrelas": estrelas, "detalhes": stats}
+
+
+# ================= NÚCLEO DE ANÁLISE =================
+
+def detectar_estrutura(audio_vec, sample_rate, duration):
+    try:
+        if duration < 30:
+            return [0.0, round(duration, 2)]
+
+        frame_size = 2048
+        hop_size = 512
+
+        rms_algo = es.RMS()
+        flux_algo = es.Flux()
+        w = es.Windowing(type="hann")
+        spec = es.Spectrum()
+
+        rms_curve = []
+        flux_curve = []
+
+        for frame in es.FrameGenerator(
+            audio_vec, frameSize=frame_size, hopSize=hop_size
+        ):
+            s = spec(w(frame))
+            rms_curve.append(rms_algo(frame))
+            flux_curve.append(flux_algo(s))
+
+        min_len = min(len(rms_curve), len(flux_curve))
+        rms_curve = np.array(rms_curve[:min_len])
+        flux_curve = np.array(flux_curve[:min_len])
+
+        if np.max(rms_curve) > 0:
+            rms_curve = rms_curve / np.max(rms_curve)
+        if np.max(flux_curve) > 0:
+            flux_curve = flux_curve / np.max(flux_curve)
+
+        novelty = rms_curve * 0.5 + flux_curve * 0.5
+        segundos_por_frame = hop_size / sample_rate
+
+        pontos_mudanca = [0.0]
+        window_frames = int(10 / segundos_por_frame)
+        step_frames = int(5 / segundos_por_frame)
+        total_frames = len(novelty)
+
+        for i in range(0, total_frames - window_frames, step_frames):
+            janela = novelty[i : i + window_frames]
+            if np.std(janela) > 0.1:
+                pico_idx = i + np.argmax(janela)
+                tempo = round(pico_idx * segundos_por_frame, 2)
+                if tempo - pontos_mudanca[-1] > 15:
+                    pontos_mudanca.append(tempo)
+
+        pontos_mudanca.append(round(duration, 2))
+        return pontos_mudanca
+
+    except Exception as e:
+        logging.error(f"Erro estrutura nova: {e}")
+        return [0.0, round(duration, 2)]
+
+
+def analisar_faixa(caminho_arquivo, embedding_model, classifier_model, classes_raw):
+    # --- AUDITORIA: INÍCIO DO TIMER ---
+    inicio_timer = time.perf_counter()
+
+    tamanho_arquivo = os.path.getsize(caminho_arquivo)
+    pode_processar, motivo = verificar_memoria_segura(tamanho_arquivo)
+
+    if not pode_processar:
+        logging.warning(f"SKIPPED {os.path.basename(caminho_arquivo)}: {motivo}")
+        return None, 0, 0
+
+    nome_arquivo = os.path.basename(caminho_arquivo)
+    chave_busca = normalizar_chave(nome_arquivo)
+    dados_humano = DATABASE_YAML.get(chave_busca)
+
+    metadata = {
+        "arquivo": nome_arquivo,
+        "caminho": caminho_arquivo,
+        "dados_projeto": {},
+        "analise_tecnica": {},
+        "analise_ia": {},
+    }
+
+    if dados_humano:
+        metadata["dados_projeto"] = {
+            "titulo": dados_humano.get("original_title") or dados_humano.get("file"),
+            "bpm_original": dados_humano.get("bpm"),
+            "tags_yaml": dados_humano.get("tags"),
+            "autor": dados_humano.get("author", "Desconhecido"),
+        }
+    else:
+        metadata["analise_tecnica"]["complexidade"] = {
+            "score_total": 0,
+            "estrelas": 0,
+            "detalhes": "YAML não encontrado",
+        }
+
+    duracao_real = 0.0
+
+    # 1. ANÁLISE TÉCNICA E EXTRAÇÃO ÚNICA (HQ 44.1kHz)
+    try:
+        # Lê do HD APENAS UMA VEZ
+        loader_hq = es.MonoLoader(filename=caminho_arquivo, sampleRate=44100)
+        audio_hq = loader_hq()
+
+        if len(audio_hq) > (MAX_DURACAO_ANALISE * 44100):
+            audio_hq = audio_hq[: int(MAX_DURACAO_ANALISE * 44100)]
+
+        duracao_real = len(audio_hq) / 44100.0
+        audio_hq_vec = essentia.array(audio_hq)
+
+        rhythm = es.RhythmExtractor2013(method="multifeature")
+        bpm, _, conf, _, _ = rhythm(audio_hq_vec)
+
+        key_ex = es.KeyExtractor()
+        key, scale, _ = key_ex(audio_hq_vec)
+
+        loudness = es.Loudness()(audio_hq_vec)
+        estrutura = detectar_estrutura(audio_hq_vec, 44100, duracao_real)
+
+        bpm_final = round(bpm, 1)
+        origem = "algoritmo"
+        if dados_humano and dados_humano.get("bpm"):
+            try:
+                bpm_str = str(dados_humano.get("bpm")).replace("'", "").replace('"', "")
+                b = float(bpm_str)
+                if b > 0:
+                    bpm_final = b
+                    origem = "yaml (humano)"
+            except:
+                pass
+
+        metadata["analise_tecnica"].update(
+            {
+                "bpm": bpm_final,
+                "origem_bpm": origem,
+                "tom": key,
+                "escala": scale,
+                "intensidade_db": round(loudness, 2),
+                "estrutura_segundos": estrutura,
+                "duracao_detectada": round(duracao_real, 2),
+            }
+        )
+
+        metadata["analise_tecnica"]["complexidade"] = (
+            calcular_complexidade(dados_humano) if dados_humano else None
+        )
+
+    except Exception as e:
+        logging.error(f"Erro math {nome_arquivo}: {e}")
+        metadata["analise_tecnica"]["erro"] = str(e)
+        return metadata, 0, 0
+
+    # 2. ANÁLISE IA (LQ 16kHz via Resample da RAM - Resolve Leitura Dupla de HD)
+    if embedding_model and classifier_model and classes_raw:
+        try:
+            resample_algo = es.Resample(inputSampleRate=44100, outputSampleRate=16000)
+            audio_lq_vec = resample_algo(audio_hq_vec)
+
+            embeddings = embedding_model(audio_lq_vec)
+            predictions = classifier_model(embeddings)
+
+            avg_preds = np.mean(predictions, axis=0)
+            top_indices = np.argsort(avg_preds)[::-1][:10]
+
+            tags_detectadas = []
+            genero_pai = "Unknown"
+
+            for i, idx in enumerate(top_indices):
+                score = float(avg_preds[idx])
+                if score < 0.05:
+                    continue
+
+                nome_full = classes_raw[idx]
+                partes = nome_full.split("---")
+                tag_limpa = partes[-1]
+
+                if i == 0:
+                    genero_pai = partes[0] if len(partes) > 0 else tag_limpa
+
+                tags_detectadas.append({"tag": tag_limpa, "score": round(score, 3)})
+
+            metadata["analise_ia"] = {
+                "genero_macro": genero_pai,
+                "estilo_principal": tags_detectadas[0]["tag"] if tags_detectadas else "Unknown",
+                "nuvem_tags": tags_detectadas,
+            }
+
+            del audio_lq_vec
+            del embeddings
+
+        except Exception as e:
+            msg = f"Erro IA Pipeline: {str(e)}"
+            logging.error(msg)
+            metadata["analise_ia"] = {"erro": msg}
+    else:
+        metadata["analise_ia"] = {"status": "Modelos nao carregados"}
+        
+    # Limpeza Limpa
+    del audio_hq
+    del audio_hq_vec
+    gc.collect()
+
+    tempo_gasto = time.perf_counter() - inicio_timer
+    return metadata, duracao_real, tempo_gasto
+
+
+# ================= MAIN (Multiprocessing) =================
+def worker_analise(caminho):
+    # Processa usando as instâncias dos modelos carregadas na thread mestre
+    global GLOBAL_EMBEDDING, GLOBAL_CLASSIFIER, GLOBAL_CLASSES
+    return analisar_faixa(caminho, GLOBAL_EMBEDDING, GLOBAL_CLASSIFIER, GLOBAL_CLASSES)
+
+def main():
+    print("\n--- INICIANDO PROCESSADOR + AUDITORIA ---")
+
+    # Referencia variávies globais do script para alimentar os childs process do pool
+    global DATABASE_YAML, GLOBAL_EMBEDDING, GLOBAL_CLASSIFIER, GLOBAL_CLASSES
+    
+    DATABASE_YAML = carregar_database_yaml()
+
+    ja_processados, lista_resultados = carregar_checkpoint()
+
+    auditor = Auditoria(ARQUIVO_AUDITORIA)
+
+    if not os.path.exists(PASTA_WAVS):
+        print("CRÍTICO: Pasta WAV não encontrada!")
+        return
+    
+    todos = [f for f in os.listdir(PASTA_WAVS) if f.lower().endswith((".wav", ".ogg"))]
+    a_fazer = [os.path.join(PASTA_WAVS, f) for f in todos if f not in ja_processados]
+
+    total = len(a_fazer)
+    print(f"Total Arquivos: {len(todos)} | Novos a Fazer: {total}")
+    print(f"Auditoria será salva em: {ARQUIVO_AUDITORIA}")
+
+    # === CARREGAMENTO MODELOS GLOBAIS ===
+    path_embed = MODELO_EMBEDDING if os.path.exists(MODELO_EMBEDDING) else os.path.join(BASE_DIR, MODELO_EMBEDDING)
+    path_class = MODELO_CLASSIFIER if os.path.exists(MODELO_CLASSIFIER) else os.path.join(BASE_DIR, MODELO_CLASSIFIER)
+    path_json = MODELO_CLASSES if os.path.exists(MODELO_CLASSES) else os.path.join(BASE_DIR, MODELO_CLASSES)
+
+    if os.path.exists(path_embed) and os.path.exists(path_class) and os.path.exists(path_json):
+        print("--- Carregando Modelos de IA ---")
+        try:
+            with open(path_json, "r") as f:
+                GLOBAL_CLASSES = json.load(f)["classes"]
+
+            print("[1/2] Carregando Extrator de Embeddings...")
+            if hasattr(es, "TensorflowPredictEffnetDiscogs"):
+                GLOBAL_EMBEDDING = es.TensorflowPredictEffnetDiscogs(graphFilename=path_embed, output="PartitionedCall:1")
+            else:
+                print(" -> Usando TensorflowPredict genérico.")
+                GLOBAL_EMBEDDING = es.TensorflowPredict(graphFilename=path_embed, input="serving_default_model_Placeholder", output="PartitionedCall:1")
+            print(" -> Embeddings carregados com sucesso.")
+
+            GLOBAL_CLASSIFIER = es.TensorflowPredict2D(graphFilename=path_class, input="serving_default_model_Placeholder", output="PartitionedCall:0")
+            print("[2/2] Classificador de Gênero carregado.")
+
+        except Exception as e:
+            print(f"ERRO CRÍTICO AO CARREGAR IA: {e}")
+            return
+    else:
+        print("Arquivos de modelo incompletos.")
+        return
+
+    if total == 0:
+        print("Tudo atualizado.")
+        return
+    
+    # ProcessPoolExecutor lidará com N filas ao mesmo tempo 
+    max_workers = max(1, multiprocessing.cpu_count() - 1)
+    print(f"-> Iniciando processamento paralelo com {max_workers} workers.")
+    contagem_sessao = 0
+
+    try:
+        with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+            resultados_futuros = {executor.submit(worker_analise, caminho): caminho for caminho in a_fazer}
+            
+            # O as_completed processa os que terminam mais rápido em vez de travar na ordem
+            for future in tqdm(concurrent.futures.as_completed(resultados_futuros), total=len(a_fazer), unit="track"):
+                try:
+                    res, duracao, tempo_gasto = future.result()
+                    caminho = resultados_futuros[future]
+                    
+                    if res:
+                        salvar_progresso(res)
+                        lista_resultados.append(res)
+                        
+                        tamanho = os.path.getsize(caminho)
+                        auditor.registrar_processamento(res['arquivo'], tamanho, duracao, tempo_gasto)
+                        
+                        contagem_sessao += 1
+                        auditor.verificar_marco(contagem_sessao)
+                        
+                except Exception as e:
+                    logging.error(f"Erro ao processar um dos arquivos no worker: {e}")
+                    
+    except KeyboardInterrupt:
+        print("\nParando paralelismo graciosamente...")
+
+    finally:
+        auditor.fechar() # Fecha o CSV com segurança
+
+    tempo_total_sessao = time.time() - auditor.inicio_global
+    print("\n--- FIM DO PROCESSAMENTO ---")
+    print(f"Tempo Total: {str(timedelta(seconds=int(tempo_total_sessao)))}")
+    print("Gerando JSON final consolidado...")
+
+    with open(ARQUIVO_FINAL_JSON, "w", encoding="utf-8") as f:
+        json.dump(lista_resultados, f, indent=4, ensure_ascii=False)
+
+    print(f"Dados JSON: {ARQUIVO_FINAL_JSON}")
+    print(f"Auditoria CSV: {ARQUIVO_AUDITORIA}")
+
+
+if __name__ == "__main__":
+    main()

scripts/classificacao/classificacao_mestre.py

@@ -0,0 +1,457 @@
+import yaml
+import json
+import os
+import logging
+import csv
+import unicodedata
+import time
+import psutil
+import glob  # Import necessário para ler múltiplos arquivos
+from datetime import datetime, timedelta
+
+# --- CONFIGURAÇÕES GERAIS ---
+BASE_DIR = "/var/www/html/trens/src_mmpSearch"
+LOG_DIR = os.path.join(BASE_DIR, "logs/classificacao_pedagogica")
+os.makedirs(LOG_DIR, exist_ok=True)
+TIMESTAMP = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+# Configuração de Log
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - CLASSIFICADOR - %(levelname)s - %(message)s",
+    handlers=[
+        logging.FileHandler(os.path.join(LOG_DIR, f"log_pedagogico_{TIMESTAMP}.log")),
+        logging.StreamHandler(),
+    ],
+)
+
+
+# --- ARQUIVOS DE ENTRADA/SAÍDA ---
+def encontrar_caminho(nome_arquivo):
+    """Procura o arquivo em locais comuns do projeto."""
+    candidatos = [
+        os.path.join(BASE_DIR, "saida_analises", nome_arquivo),
+        os.path.join(BASE_DIR, "data", nome_arquivo),
+        os.path.join(BASE_DIR, nome_arquivo),
+        nome_arquivo,
+    ]
+    for c in candidatos:
+        if os.path.exists(c):
+            return c
+    return nome_arquivo
+
+
+# --- MUDANÇA AQUI: Aponta para a pasta dos lotes ---
+PASTA_LOTES_YAML = os.path.join(BASE_DIR, "lotes_yaml")
+# ARQUIVO_YAML = "all.yml"  <-- Removido/Comentado
+
+ARQUIVO_AUDIO_INPUT = encontrar_caminho("db_final_completo.json")
+ARQUIVO_FINAL_OUTPUT = os.path.join(
+    BASE_DIR, "saida_analises", "db_projetos_classificados.json"
+)
+ARQUIVO_AUDITORIA = os.path.join(
+    BASE_DIR, "saida_analises", "audit_classificacao_pedagogica.csv"
+)
+
+# --- LISTAS DE REFERÊNCIA ---
+PLUGINS_NATIVOS = [
+    "tripleoscillator",
+    "zynaddsubfx",
+    "sfxr",
+    "organic",
+    "audiofileprocessor",
+    "lb302",
+    "kicker",
+    "watsyn",
+    "bitinvader",
+    "freeboy",
+    "mallets",
+    "vibed",
+]
+
+
+# --- CLASSE DE AUDITORIA DE PERFORMANCE ---
+class AuditoriaPerformance:
+    def __init__(self):
+        self.inicio_global = time.time()
+        self.process = psutil.Process(os.getpid())
+        self.marcos = [1, 10, 100, 500, 1000, 2000, 5000]
+
+    def get_metricas(self):
+        """Retorna uso atual de RAM (MB) e CPU (%)"""
+        try:
+            ram_mb = self.process.memory_info().rss / (1024 * 1024)
+            cpu_pct = self.process.cpu_percent(interval=None)
+            return round(ram_mb, 2), cpu_pct
+        except:
+            return 0.0, 0.0
+
+    def verificar_marco(self, contagem):
+        if contagem in self.marcos:
+            tempo_decorrido = time.time() - self.inicio_global
+            logging.info(
+                f"--- MARCO: {contagem} projetos processados em {str(timedelta(seconds=int(tempo_decorrido)))} ---"
+            )
+
+
+# --- FUNÇÕES AUXILIARES ---
+
+
+def normalizar_chave(texto):
+    if not texto:
+        return ""
+    s = str(texto)
+    if "." in s:
+        s = os.path.splitext(s)[0]
+    s = unicodedata.normalize("NFKD", s).encode("ASCII", "ignore").decode("ASCII")
+    return s.lower().replace(" ", "").replace("-", "").replace("_", "").strip()
+
+
+def calcular_nivel_ponderado(
+    num_tracks, tem_automacao, usa_zyn, qtd_secoes_audio, fx_criativos
+):
+    pontos = 0
+
+    # 1. Volume
+    if num_tracks >= 8:
+        pontos += 1
+    if num_tracks >= 16:
+        pontos += 2
+    if num_tracks >= 32:
+        pontos += 3
+    if num_tracks >= 64:
+        pontos += 4    
+    elif num_tracks >= 128:
+        pontos += 5
+
+    # 2. Técnica
+    if tem_automacao:
+        pontos += 2
+    if usa_zyn:
+        pontos += 1.5
+    if fx_criativos:
+        pontos += 1
+
+    # 3. Estrutura (Áudio)
+    if qtd_secoes_audio >= 3:
+        pontos += 1.5
+    elif qtd_secoes_audio >= 5:
+        pontos += 3
+    elif qtd_secoes_audio >= 10:
+        pontos += 5
+
+    if pontos < 3:
+        return "Iniciante"
+    if pontos < 6:
+        return "Intermediário"
+    return "Avançado"
+
+
+def analisar_tags_pedagogicas(tracks, bpm, qtd_secoes):
+    tags = []
+    tem_kicker = False
+    tem_zyn = False
+    tem_automacao = False
+    tem_fx_chain = False
+    rolagem_trap = False
+
+    for t in tracks:
+        inst = t.get("instrumenttrack")
+        if not inst:
+            continue
+
+        nome = inst.get("name", "").lower()
+        plugin = inst.get("plugin_name", "").lower()
+
+        if plugin == "kicker":
+            tem_kicker = True
+        if plugin == "zynaddsubfx":
+            tem_zyn = True
+        if "automation" in str(inst).lower():
+            tem_automacao = True
+
+        fx = inst.get("fxchain", {})
+        if fx and int(fx.get("numofeffects", 0)) > 0:
+            tem_fx_chain = True
+
+        patterns = inst.get("patterns", [])
+        if patterns:
+            for pat in patterns:
+                steps = pat.get("steps", [])
+                consecutivos = 0
+                for step in steps:
+                    val = 1 if step else 0
+                    consecutivos = (consecutivos + 1) if val else 0
+                    if consecutivos >= 3 and ("hat" in nome):
+                        rolagem_trap = True
+
+    if tem_kicker:
+        tags.append("Síntese de Bateria")
+    if tem_zyn:
+        tags.append("Design de Som Avançado")
+    if tem_automacao:
+        tags.append("Automação")
+    if tem_fx_chain:
+        tags.append("Mixagem com FX")
+
+    if rolagem_trap and bpm > 115:
+        tags.append("Trap Rhythm")
+    if bpm < 100 and not rolagem_trap:
+        tags.append("Boombap Groove")
+
+    if qtd_secoes >= 4:
+        tags.append("Arranjo Completo")
+    elif qtd_secoes <= 1:
+        tags.append("Loop Estático")
+    else:
+        tags.append("Estrutura Simples")
+
+    return tags, tem_automacao, tem_zyn, tem_fx_chain
+
+
+# --- FUNÇÃO PARA CARREGAR LOTES YAML ---
+def carregar_todos_yamls(pasta_lotes):
+    """Lê todos os arquivos YAML da pasta especificada e retorna uma lista única."""
+    logging.info(f"Buscando arquivos YAML em: {pasta_lotes}")
+
+    padrao = os.path.join(pasta_lotes, "*.yml")
+    arquivos = sorted(glob.glob(padrao))
+
+    todos_projetos = []
+
+    if not arquivos:
+        logging.warning("Nenhum arquivo YAML encontrado na pasta de lotes.")
+        # Tenta fallback para o arquivo único antigo se existir
+        caminho_unico = encontrar_caminho("all.yml")
+        if os.path.exists(caminho_unico):
+            logging.info(f"Fallback: Carregando arquivo único {caminho_unico}")
+            with open(caminho_unico, "r", encoding="utf-8") as f:
+                return yaml.safe_load(f)
+        return []
+
+    for arq in arquivos:
+        try:
+            logging.info(f"Carregando lote: {os.path.basename(arq)}")
+            with open(arq, "r", encoding="utf-8") as f:
+                lote = yaml.safe_load(f)
+                if lote:
+                    # Garante que seja uma lista
+                    if isinstance(lote, list):
+                        todos_projetos.extend(lote)
+                    elif isinstance(lote, dict):
+                        todos_projetos.append(lote)
+        except Exception as e:
+            logging.error(f"Erro ao ler arquivo {arq}: {e}")
+
+    logging.info(f"Total de projetos carregados dos lotes: {len(todos_projetos)}")
+    return todos_projetos
+
+
+# --- MAIN ---
+
+
+def main():
+    logging.info("--- INICIANDO CLASSIFICADOR PEDAGÓGICO (COM AUDITORIA E LOTES) ---")
+
+    auditor = AuditoriaPerformance()
+
+    # 1. Carregar YAML (Adaptado para Lotes)
+    if not os.path.exists(PASTA_LOTES_YAML):
+        logging.warning(f"Pasta de lotes não encontrada: {PASTA_LOTES_YAML}")
+        # Tenta criar se não existir (embora deva existir com arquivos)
+        # os.makedirs(PASTA_LOTES_YAML, exist_ok=True)
+
+    dados_projetos = carregar_todos_yamls(PASTA_LOTES_YAML)
+
+    if not dados_projetos:
+        logging.critical("Nenhum dado de projeto carregado. Encerrando.")
+        return
+
+    # 2. Carregar JSON de Áudio
+    lookup_audio = {}
+    total_audio_carregado = 0
+
+    if os.path.exists(ARQUIVO_AUDIO_INPUT):
+        logging.info(f"Lendo Análise de Áudio: {ARQUIVO_AUDIO_INPUT}")
+        with open(ARQUIVO_AUDIO_INPUT, "r", encoding="utf-8") as f:
+            lista_audio = json.load(f)
+            for item in lista_audio:
+                chave = normalizar_chave(item.get("arquivo", ""))
+                if chave:
+                    lookup_audio[chave] = item
+            total_audio_carregado = len(lista_audio)
+    else:
+        logging.warning(f"Arquivo de áudio {ARQUIVO_AUDIO_INPUT} não encontrado.")
+
+    db_final = []
+    estatisticas = {"Iniciante": 0, "Intermediário": 0, "Avançado": 0, "Total": 0}
+
+    # --- PREPARAR CSV DE AUDITORIA ---
+    with open(ARQUIVO_AUDITORIA, "w", newline="", encoding="utf-8") as csvfile:
+        writer = csv.writer(csvfile)
+        # Cabeçalho expandido com métricas de performance
+        writer.writerow(
+            [
+                "Timestamp",
+                "ID_Projeto",
+                "Audio_Encontrado",
+                "Qtd_Secoes",
+                "Nivel",
+                "Genero_Final",
+                "Tempo_Proc_ms",
+                "RAM_Uso_MB",
+                "Tags",
+            ]
+        )
+
+        logging.info(f"Processando {len(dados_projetos)} projetos...")
+
+        contagem_processados = 0
+
+        for proj in dados_projetos:
+            if not proj or not isinstance(proj, dict):
+                continue
+
+            # --- INÍCIO TIMER INDIVIDUAL ---
+            inicio_item = time.perf_counter()
+
+            nome_arquivo = proj.get("file", "")
+            if not nome_arquivo:
+                continue
+
+            try:
+                # Lógica de Classificação
+                chave_proj = normalizar_chave(nome_arquivo)
+                dados_audio = lookup_audio.get(chave_proj, {})
+
+                analise_tec = dados_audio.get("analise_tecnica", {})
+                analise_ia = dados_audio.get("analise_ia", {})
+
+                raw_bpm = proj.get("bpm", 120)
+                try:
+                    bpm_projeto = float(raw_bpm)
+                except:
+                    bpm_projeto = 120.0
+
+                est_seg = analise_tec.get("estrutura_segundos", [])
+                qtd_secoes = len(est_seg) if isinstance(est_seg, list) else 1
+
+                tracks = proj.get("tracks", [])
+                tags, tem_auto, tem_zyn, tem_fx = analisar_tags_pedagogicas(
+                    tracks, bpm_projeto, qtd_secoes
+                )
+
+                plugins_usados = set()
+                usa_vst_externo = False
+                for t in tracks:
+                    inst_track = t.get("instrumenttrack", {})
+                    if not inst_track:
+                        continue
+                    p = inst_track.get("plugin_name", "")
+                    if p:
+                        p_norm = p.lower()
+                        plugins_usados.add(p_norm)
+                        if p_norm not in PLUGINS_NATIVOS:
+                            usa_vst_externo = True
+
+                nivel = calcular_nivel_ponderado(
+                    len(tracks), tem_auto, tem_zyn, qtd_secoes, tem_fx
+                )
+                estatisticas[nivel] += 1
+                estatisticas["Total"] += 1
+
+                genero = "Desconhecido"
+                if analise_ia and "estilo_principal" in analise_ia:
+                    genero = analise_ia["estilo_principal"]
+                elif (
+                    "genero_macro" in analise_ia
+                    and analise_ia["genero_macro"] != "Unknown"
+                ):
+                    genero = analise_ia["genero_macro"]
+
+                if genero == "Desconhecido" or genero == "Unknown":
+                    if "Trap Rhythm" in tags:
+                        genero = "Trap (Rule-based)"
+                    elif bpm_projeto > 120:
+                        genero = "Electronic (Rule-based)"
+                    else:
+                        genero = "HipHop/Downtempo (Rule-based)"
+
+                # Objeto Final
+                item = {
+                    "id": nome_arquivo,
+                    "titulo": proj.get("original_title", nome_arquivo),
+                    "match_audio": bool(dados_audio),
+                    "pedagogia": {
+                        "nivel": nivel,
+                        "tags_aprendizado": tags,
+                        "plugins_principais": list(plugins_usados),
+                        "requer_vst_externo": usa_vst_externo,
+                    },
+                    "tecnica": {
+                        "bpm": bpm_projeto,
+                        "tom": analise_tec.get("tom", "N/A"),
+                        "escala": analise_tec.get("escala", "N/A"),
+                        "estrutura_qtd_secoes": qtd_secoes,
+                        "intensidade_media": analise_tec.get("intensidade_db", "N/A"),
+                    },
+                    "classificacao": {
+                        "genero": genero,
+                        "mood": "Energético"
+                        if analise_tec.get("intensidade_db", -20) > -10
+                        else "Suave",
+                    },
+                }
+
+                db_final.append(item)
+
+                # --- FIM TIMER INDIVIDUAL ---
+                tempo_ms = (time.perf_counter() - inicio_item) * 1000
+                ram_atual, _ = auditor.get_metricas()
+                ts_agora = datetime.now().strftime("%H:%M:%S")
+
+                # Registra Auditoria com Performance
+                writer.writerow(
+                    [
+                        ts_agora,
+                        nome_arquivo,
+                        "SIM" if dados_audio else "NAO",
+                        qtd_secoes,
+                        nivel,
+                        genero,
+                        f"{tempo_ms:.3f}",  # Tempo em milissegundos com 3 casas
+                        f"{ram_atual:.2f}",  # RAM em MB
+                        "|".join(tags),
+                    ]
+                )
+
+                contagem_processados += 1
+                auditor.verificar_marco(contagem_processados)
+
+            except Exception as e:
+                logging.error(
+                    f"Erro ao processar projeto '{proj.get('file', 'unknown')}': {e}"
+                )
+                continue
+
+    # Salvar Resultado JSON
+    logging.info(f"Salvando JSON Final: {ARQUIVO_FINAL_OUTPUT}")
+    with open(ARQUIVO_FINAL_OUTPUT, "w", encoding="utf-8") as f:
+        json.dump(db_final, f, indent=4, ensure_ascii=False)
+
+    tempo_total = time.time() - auditor.inicio_global
+
+    logging.info("--- RELATÓRIO FINAL ---")
+    logging.info(f"Tempo Total de Execução: {str(timedelta(seconds=int(tempo_total)))}")
+    logging.info(f"Total Processado: {estatisticas['Total']}")
+    logging.info(f"Iniciantes: {estatisticas['Iniciante']}")
+    logging.info(f"Intermediários: {estatisticas['Intermediário']}")
+    logging.info(f"Avançados: {estatisticas['Avançado']}")
+    logging.info(
+        f"Taxa de Casamento com Áudio: {len(lookup_audio)}/{total_audio_carregado}"
+    )
+    logging.info(f"Auditoria salva em: {ARQUIVO_AUDITORIA}")
+
+
+if __name__ == "__main__":
+    main()

scripts/classificacao/classificador_mestre.py

@@ -0,0 +1,274 @@
+import yaml
+import json
+import os
+import logging
+
+# --- CONFIGURAÇÕES DE LOG ---
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - CLASSIFICADOR - %(levelname)s - %(message)s",
+    handlers=[
+        logging.FileHandler("classificacao_pedagogica.log"),
+        logging.StreamHandler(),
+    ],
+)
+
+# --- CONFIGURAÇÕES ---
+ARQUIVO_YAML = "all.yml"
+ARQUIVO_AUDIO = "analise_audio.json"
+ARQUIVO_FINAL = "db_projetos_classificados.json"
+
+PLUGINS_NATIVOS = [
+    "tripleoscillator",
+    "zynaddsubfx",
+    "sfxr",
+    "organic",
+    "audiofileprocessor",
+    "lb302",
+    "kicker",
+    "watsyn",
+    "bitinvader",
+    "freeboy",
+    "mallets",
+    "vibed",
+]
+
+# --- LÓGICA PEDAGÓGICA AVANÇADA ---
+
+
+def calcular_nivel_ponderado(
+    num_tracks, tem_automacao, plugins_complexos, qtd_secoes_audio, fx_criativos
+):
+    """
+    Calcula nível pedagógico (Iniciante, Intermediário, Avançado)
+    baseado em esforço de engenharia e complexidade musical.
+    """
+    pontos = 0
+
+    # 1. Volume de Trabalho
+    if num_tracks >= 8:
+        pontos += 1
+    if num_tracks >= 16:
+        pontos += 1
+
+    # 2. Profundidade Técnica
+    if tem_automacao:
+        pontos += 2  # Automação é sinal forte de polimento
+    if plugins_complexos:
+        pontos += 1.5  # Usar ZynAddSubFx requer estudo
+    if fx_criativos:
+        pontos += 1  # Usar efeitos além do básico
+
+    # 3. Estrutura Musical (Vinda do Essentia)
+    # Se o áudio tem > 3 seções detectadas, há arranjo (não é só loop)
+    if qtd_secoes_audio >= 3:
+        pontos += 2
+    elif qtd_secoes_audio >= 5:
+        pontos += 3
+
+    # Classificação
+    if pontos < 3:
+        return "Iniciante"
+    if pontos < 6:
+        return "Intermediário"
+    return "Avançado"
+
+
+def analisar_tags_pedagogicas(tracks, bpm, qtd_secoes, tom, escala):
+    tags = []
+
+    # Flags de estado técnico
+    tem_zyn = False
+    tem_automacao = False
+    tem_fx_chain = False
+    qtd_nativos = 0
+    qtd_vst_externos = 0
+    tem_audio_processor = False
+
+    for t in tracks:
+        inst = t.get("instrumenttrack")
+        if not inst:
+            continue
+
+        plugin = inst.get("plugin_name", "").lower()
+
+        # Mapeamento de Plugins e Dependências
+        if plugin in PLUGINS_NATIVOS:
+            qtd_nativos += 1
+        elif plugin == "vestige":
+            qtd_vst_externos += 1
+
+        if plugin == "zynaddsubfx":
+            tem_zyn = True
+
+        if plugin == "audiofileprocessor":
+            tem_audio_processor = True
+
+        if "automation" in str(inst).lower():
+            tem_automacao = True
+
+        fx = inst.get("fxchain", {})
+        if fx and int(fx.get("numofeffects", 0)) > 0:
+            tem_fx_chain = True
+
+    # --- 1. Estética da Escassez e Modulação Marginal ---
+    if qtd_nativos > 0 and qtd_vst_externos == 0:
+        tags.append("Estética da Escassez: Síntese Nativa")
+
+    if tem_automacao and qtd_vst_externos == 0:
+        tags.append("Design de Som Marginal (Automação como Timbre)")
+
+    # --- 2. Taxonomia de Bloom (Carga Cognitiva) ---
+    if tem_zyn or tem_fx_chain:
+        tags.append("Bloom: Criação (Sound Design Complexo)")
+    elif tem_audio_processor and not tem_automacao:
+        tags.append("Bloom: Aplicação (Sample Inalterado)")
+
+    # --- 3. Fluência Estrutural (Macroforma Musical) ---
+    if qtd_secoes >= 3:
+        tags.append("Fluência Estrutural: Arranjo Completo (Macroforma)")
+    else:
+        tags.append("Fluência Estrutural: Loop Estático")
+
+    # --- 4. Acessibilidade Ergonômica (Música de Computador) ---
+    if tom == "A" and escala == "minor":
+        tags.append("Ergonomia Autodidata (Teclado QWERTY / Lá Menor)")
+
+    # --- 5. Herança Cultural / Andamento ---
+    if 80 <= bpm <= 95:
+        tags.append("Cadência Clássica (Boom Bap)")
+    elif bpm >= 130:
+        tags.append("Estética Trap/Funk/Eletrônica / Alta Energia")
+
+    return tags
+
+
+def main():
+    logging.info("Iniciando Classificador Mestre...")
+
+    # 1. Carregar YAML
+    if not os.path.exists(ARQUIVO_YAML):
+        logging.critical(f"{ARQUIVO_YAML} não encontrado.")
+        return
+
+    with open(ARQUIVO_YAML, "r", encoding="utf-8") as f:
+        dados_projetos = yaml.safe_load(f)
+
+    # 2. Carregar JSON de Áudio
+    lookup_audio = {}
+    if os.path.exists(ARQUIVO_AUDIO):
+        with open(ARQUIVO_AUDIO, "r", encoding="utf-8") as f:
+            lista_audio = json.load(f)
+            for item in lista_audio:
+                # Normaliza chave de busca (nome do arquivo sem extensão)
+                nome_base = os.path.splitext(item["arquivo"])[0]
+                lookup_audio[nome_base] = item
+    else:
+        logging.warning(
+            "JSON de áudio não encontrado. Classificação será apenas estática."
+        )
+
+    db_final = []
+
+    logging.info(f"Processando {len(dados_projetos)} entradas do YAML...")
+
+    for proj in dados_projetos:
+        if not proj:
+            continue
+
+        try:
+            nome_arquivo = proj.get("file", "")
+            if not nome_arquivo:
+                continue
+
+            # Tenta casar com dados de áudio
+            # Nota: Seu YAML usa nomes como 'trap-fyrebreak', o arquivo de audio deve ser 'trap-fyrebreak.wav'
+            # O lookup_audio já está sem extensão.
+            dados_audio = lookup_audio.get(nome_arquivo, {})
+            analise_tec = dados_audio.get("analise_tecnica", {})
+            analise_ia = dados_audio.get("analise_ia", {})
+
+            # Dados Técnicos
+            raw_bpm = proj.get("bpm", 120)
+            bpm_projeto = (
+                float(raw_bpm) if str(raw_bpm).replace(".", "", 1).isdigit() else 120.0
+            )
+
+            # Feature nova: Quantidade de seções (Intro, Verso...)
+            qtd_secoes = analise_tec.get("qtd_secoes_detectadas", 1)
+
+            # Análise Estática (Tracks e Plugins)
+            tracks = proj.get("tracks", [])
+            tags, tem_auto, tem_zyn, tem_fx = analisar_tags_pedagogicas(
+                tracks=proj.get("tracks", []),
+                bpm=bpm_projeto,
+                qtd_secoes=qtd_secoes,
+                tom=analise_tec.get("tom", "N/A"),
+                escala=analise_tec.get("escala", "N/A"),
+            )
+
+            # Verifica plugins externos
+            plugins_usados = set()
+            usa_vst_externo = False
+            for t in tracks:
+                p = t.get("instrumenttrack", {}).get("plugin_name", "")
+                if p:
+                    plugins_usados.add(p)
+                    if p.lower() not in PLUGINS_NATIVOS:
+                        usa_vst_externo = True
+
+            # Cálculo de Nível (Nova Lógica)
+            nivel = calcular_nivel_ponderado(
+                len(tracks), tem_auto, tem_zyn, qtd_secoes, tem_fx
+            )
+
+            # Define Gênero (IA tem prioridade, depois regras manuais)
+            genero = analise_ia.get("genero_predito", "Desconhecido")
+            if genero == "Desconhecido" or analise_ia.get("confianca_genero", 0) < 0.4:
+                # Fallback manual simples
+                if "Trap Rhythm" in tags:
+                    genero = "Trap"
+                elif bpm_projeto > 120:
+                    genero = "Electronic"
+                else:
+                    genero = "HipHop/Downtempo"
+
+            # Montagem do Objeto
+            item = {
+                "id": nome_arquivo,
+                "titulo": proj.get("original_title", nome_arquivo),
+                "pedagogia": {
+                    "nivel": nivel,
+                    "tags_aprendizado": tags,
+                    "plugins_principais": list(plugins_usados),
+                    "requer_vst_externo": usa_vst_externo,
+                },
+                "tecnica": {
+                    "bpm": bpm_projeto,
+                    "tom": analise_tec.get("tom", "N/A"),
+                    "escala": analise_tec.get("escala", "N/A"),
+                    "estrutura_detectada": f"{qtd_secoes} seções distintas",
+                },
+                "classificacao": {
+                    "genero": genero,
+                    "mood": "Energético"
+                    if analise_tec.get("intensidade", -20) > -10
+                    else "Suave",
+                },
+            }
+
+            db_final.append(item)
+
+        except Exception as e:
+            logging.error(f"Erro ao processar projeto '{proj.get('file')}': {e}")
+            continue
+
+    # Salvar Resultado
+    with open(ARQUIVO_FINAL, "w", encoding="utf-8") as f:
+        json.dump(db_final, f, indent=4, ensure_ascii=False)
+
+    logging.info(f"Sucesso! {len(db_final)} projetos classificados e salvos.")
+
+
+if __name__ == "__main__":
+    main()

scripts/classificacao/discogs-effnet-bs64-1.json

@@ -0,0 +1,468 @@
+{
+    "name": "EffnetDiscogs",
+    "type": "Music style classification and embeddings",
+    "link": "https://essentia.upf.edu/models/music-style-classification/discogs-effnet/discogs-effnet-bs64-1.pb",
+    "version": "1",
+    "description": "Prediction of the top-400 music styles in the Discogs-4M dataset (unreleased)",
+    "author": "Pablo Alonso",
+    "email": "pablo.alonso@upf.edu",
+    "release_date": "2022-02-17",
+    "framework": "tensorflow",
+    "framework_version": "2.8.0",
+    "classes": [
+        "Blues---Boogie Woogie",
+        "Blues---Chicago Blues",
+        "Blues---Country Blues",
+        "Blues---Delta Blues",
+        "Blues---Electric Blues",
+        "Blues---Harmonica Blues",
+        "Blues---Jump Blues",
+        "Blues---Louisiana Blues",
+        "Blues---Modern Electric Blues",
+        "Blues---Piano Blues",
+        "Blues---Rhythm & Blues",
+        "Blues---Texas Blues",
+        "Brass & Military---Brass Band",
+        "Brass & Military---Marches",
+        "Brass & Military---Military",
+        "Children's---Educational",
+        "Children's---Nursery Rhymes",
+        "Children's---Story",
+        "Classical---Baroque",
+        "Classical---Choral",
+        "Classical---Classical",
+        "Classical---Contemporary",
+        "Classical---Impressionist",
+        "Classical---Medieval",
+        "Classical---Modern",
+        "Classical---Neo-Classical",
+        "Classical---Neo-Romantic",
+        "Classical---Opera",
+        "Classical---Post-Modern",
+        "Classical---Renaissance",
+        "Classical---Romantic",
+        "Electronic---Abstract",
+        "Electronic---Acid",
+        "Electronic---Acid House",
+        "Electronic---Acid Jazz",
+        "Electronic---Ambient",
+        "Electronic---Bassline",
+        "Electronic---Beatdown",
+        "Electronic---Berlin-School",
+        "Electronic---Big Beat",
+        "Electronic---Bleep",
+        "Electronic---Breakbeat",
+        "Electronic---Breakcore",
+        "Electronic---Breaks",
+        "Electronic---Broken Beat",
+        "Electronic---Chillwave",
+        "Electronic---Chiptune",
+        "Electronic---Dance-pop",
+        "Electronic---Dark Ambient",
+        "Electronic---Darkwave",
+        "Electronic---Deep House",
+        "Electronic---Deep Techno",
+        "Electronic---Disco",
+        "Electronic---Disco Polo",
+        "Electronic---Donk",
+        "Electronic---Downtempo",
+        "Electronic---Drone",
+        "Electronic---Drum n Bass",
+        "Electronic---Dub",
+        "Electronic---Dub Techno",
+        "Electronic---Dubstep",
+        "Electronic---Dungeon Synth",
+        "Electronic---EBM",
+        "Electronic---Electro",
+        "Electronic---Electro House",
+        "Electronic---Electroclash",
+        "Electronic---Euro House",
+        "Electronic---Euro-Disco",
+        "Electronic---Eurobeat",
+        "Electronic---Eurodance",
+        "Electronic---Experimental",
+        "Electronic---Freestyle",
+        "Electronic---Future Jazz",
+        "Electronic---Gabber",
+        "Electronic---Garage House",
+        "Electronic---Ghetto",
+        "Electronic---Ghetto House",
+        "Electronic---Glitch",
+        "Electronic---Goa Trance",
+        "Electronic---Grime",
+        "Electronic---Halftime",
+        "Electronic---Hands Up",
+        "Electronic---Happy Hardcore",
+        "Electronic---Hard House",
+        "Electronic---Hard Techno",
+        "Electronic---Hard Trance",
+        "Electronic---Hardcore",
+        "Electronic---Hardstyle",
+        "Electronic---Hi NRG",
+        "Electronic---Hip Hop",
+        "Electronic---Hip-House",
+        "Electronic---House",
+        "Electronic---IDM",
+        "Electronic---Illbient",
+        "Electronic---Industrial",
+        "Electronic---Italo House",
+        "Electronic---Italo-Disco",
+        "Electronic---Italodance",
+        "Electronic---Jazzdance",
+        "Electronic---Juke",
+        "Electronic---Jumpstyle",
+        "Electronic---Jungle",
+        "Electronic---Latin",
+        "Electronic---Leftfield",
+        "Electronic---Makina",
+        "Electronic---Minimal",
+        "Electronic---Minimal Techno",
+        "Electronic---Modern Classical",
+        "Electronic---Musique Concr\u00e8te",
+        "Electronic---Neofolk",
+        "Electronic---New Age",
+        "Electronic---New Beat",
+        "Electronic---New Wave",
+        "Electronic---Noise",
+        "Electronic---Nu-Disco",
+        "Electronic---Power Electronics",
+        "Electronic---Progressive Breaks",
+        "Electronic---Progressive House",
+        "Electronic---Progressive Trance",
+        "Electronic---Psy-Trance",
+        "Electronic---Rhythmic Noise",
+        "Electronic---Schranz",
+        "Electronic---Sound Collage",
+        "Electronic---Speed Garage",
+        "Electronic---Speedcore",
+        "Electronic---Synth-pop",
+        "Electronic---Synthwave",
+        "Electronic---Tech House",
+        "Electronic---Tech Trance",
+        "Electronic---Techno",
+        "Electronic---Trance",
+        "Electronic---Tribal",
+        "Electronic---Tribal House",
+        "Electronic---Trip Hop",
+        "Electronic---Tropical House",
+        "Electronic---UK Garage",
+        "Electronic---Vaporwave",
+        "Folk, World, & Country---African",
+        "Folk, World, & Country---Bluegrass",
+        "Folk, World, & Country---Cajun",
+        "Folk, World, & Country---Canzone Napoletana",
+        "Folk, World, & Country---Catalan Music",
+        "Folk, World, & Country---Celtic",
+        "Folk, World, & Country---Country",
+        "Folk, World, & Country---Fado",
+        "Folk, World, & Country---Flamenco",
+        "Folk, World, & Country---Folk",
+        "Folk, World, & Country---Gospel",
+        "Folk, World, & Country---Highlife",
+        "Folk, World, & Country---Hillbilly",
+        "Folk, World, & Country---Hindustani",
+        "Folk, World, & Country---Honky Tonk",
+        "Folk, World, & Country---Indian Classical",
+        "Folk, World, & Country---La\u00efk\u00f3",
+        "Folk, World, & Country---Nordic",
+        "Folk, World, & Country---Pacific",
+        "Folk, World, & Country---Polka",
+        "Folk, World, & Country---Ra\u00ef",
+        "Folk, World, & Country---Romani",
+        "Folk, World, & Country---Soukous",
+        "Folk, World, & Country---S\u00e9ga",
+        "Folk, World, & Country---Volksmusik",
+        "Folk, World, & Country---Zouk",
+        "Folk, World, & Country---\u00c9ntekhno",
+        "Funk / Soul---Afrobeat",
+        "Funk / Soul---Boogie",
+        "Funk / Soul---Contemporary R&B",
+        "Funk / Soul---Disco",
+        "Funk / Soul---Free Funk",
+        "Funk / Soul---Funk",
+        "Funk / Soul---Gospel",
+        "Funk / Soul---Neo Soul",
+        "Funk / Soul---New Jack Swing",
+        "Funk / Soul---P.Funk",
+        "Funk / Soul---Psychedelic",
+        "Funk / Soul---Rhythm & Blues",
+        "Funk / Soul---Soul",
+        "Funk / Soul---Swingbeat",
+        "Funk / Soul---UK Street Soul",
+        "Hip Hop---Bass Music",
+        "Hip Hop---Boom Bap",
+        "Hip Hop---Bounce",
+        "Hip Hop---Britcore",
+        "Hip Hop---Cloud Rap",
+        "Hip Hop---Conscious",
+        "Hip Hop---Crunk",
+        "Hip Hop---Cut-up/DJ",
+        "Hip Hop---DJ Battle Tool",
+        "Hip Hop---Electro",
+        "Hip Hop---G-Funk",
+        "Hip Hop---Gangsta",
+        "Hip Hop---Grime",
+        "Hip Hop---Hardcore Hip-Hop",
+        "Hip Hop---Horrorcore",
+        "Hip Hop---Instrumental",
+        "Hip Hop---Jazzy Hip-Hop",
+        "Hip Hop---Miami Bass",
+        "Hip Hop---Pop Rap",
+        "Hip Hop---Ragga HipHop",
+        "Hip Hop---RnB/Swing",
+        "Hip Hop---Screw",
+        "Hip Hop---Thug Rap",
+        "Hip Hop---Trap",
+        "Hip Hop---Trip Hop",
+        "Hip Hop---Turntablism",
+        "Jazz---Afro-Cuban Jazz",
+        "Jazz---Afrobeat",
+        "Jazz---Avant-garde Jazz",
+        "Jazz---Big Band",
+        "Jazz---Bop",
+        "Jazz---Bossa Nova",
+        "Jazz---Contemporary Jazz",
+        "Jazz---Cool Jazz",
+        "Jazz---Dixieland",
+        "Jazz---Easy Listening",
+        "Jazz---Free Improvisation",
+        "Jazz---Free Jazz",
+        "Jazz---Fusion",
+        "Jazz---Gypsy Jazz",
+        "Jazz---Hard Bop",
+        "Jazz---Jazz-Funk",
+        "Jazz---Jazz-Rock",
+        "Jazz---Latin Jazz",
+        "Jazz---Modal",
+        "Jazz---Post Bop",
+        "Jazz---Ragtime",
+        "Jazz---Smooth Jazz",
+        "Jazz---Soul-Jazz",
+        "Jazz---Space-Age",
+        "Jazz---Swing",
+        "Latin---Afro-Cuban",
+        "Latin---Bai\u00e3o",
+        "Latin---Batucada",
+        "Latin---Beguine",
+        "Latin---Bolero",
+        "Latin---Boogaloo",
+        "Latin---Bossanova",
+        "Latin---Cha-Cha",
+        "Latin---Charanga",
+        "Latin---Compas",
+        "Latin---Cubano",
+        "Latin---Cumbia",
+        "Latin---Descarga",
+        "Latin---Forr\u00f3",
+        "Latin---Guaguanc\u00f3",
+        "Latin---Guajira",
+        "Latin---Guaracha",
+        "Latin---MPB",
+        "Latin---Mambo",
+        "Latin---Mariachi",
+        "Latin---Merengue",
+        "Latin---Norte\u00f1o",
+        "Latin---Nueva Cancion",
+        "Latin---Pachanga",
+        "Latin---Porro",
+        "Latin---Ranchera",
+        "Latin---Reggaeton",
+        "Latin---Rumba",
+        "Latin---Salsa",
+        "Latin---Samba",
+        "Latin---Son",
+        "Latin---Son Montuno",
+        "Latin---Tango",
+        "Latin---Tejano",
+        "Latin---Vallenato",
+        "Non-Music---Audiobook",
+        "Non-Music---Comedy",
+        "Non-Music---Dialogue",
+        "Non-Music---Education",
+        "Non-Music---Field Recording",
+        "Non-Music---Interview",
+        "Non-Music---Monolog",
+        "Non-Music---Poetry",
+        "Non-Music---Political",
+        "Non-Music---Promotional",
+        "Non-Music---Radioplay",
+        "Non-Music---Religious",
+        "Non-Music---Spoken Word",
+        "Pop---Ballad",
+        "Pop---Bollywood",
+        "Pop---Bubblegum",
+        "Pop---Chanson",
+        "Pop---City Pop",
+        "Pop---Europop",
+        "Pop---Indie Pop",
+        "Pop---J-pop",
+        "Pop---K-pop",
+        "Pop---Kay\u014dkyoku",
+        "Pop---Light Music",
+        "Pop---Music Hall",
+        "Pop---Novelty",
+        "Pop---Parody",
+        "Pop---Schlager",
+        "Pop---Vocal",
+        "Reggae---Calypso",
+        "Reggae---Dancehall",
+        "Reggae---Dub",
+        "Reggae---Lovers Rock",
+        "Reggae---Ragga",
+        "Reggae---Reggae",
+        "Reggae---Reggae-Pop",
+        "Reggae---Rocksteady",
+        "Reggae---Roots Reggae",
+        "Reggae---Ska",
+        "Reggae---Soca",
+        "Rock---AOR",
+        "Rock---Acid Rock",
+        "Rock---Acoustic",
+        "Rock---Alternative Rock",
+        "Rock---Arena Rock",
+        "Rock---Art Rock",
+        "Rock---Atmospheric Black Metal",
+        "Rock---Avantgarde",
+        "Rock---Beat",
+        "Rock---Black Metal",
+        "Rock---Blues Rock",
+        "Rock---Brit Pop",
+        "Rock---Classic Rock",
+        "Rock---Coldwave",
+        "Rock---Country Rock",
+        "Rock---Crust",
+        "Rock---Death Metal",
+        "Rock---Deathcore",
+        "Rock---Deathrock",
+        "Rock---Depressive Black Metal",
+        "Rock---Doo Wop",
+        "Rock---Doom Metal",
+        "Rock---Dream Pop",
+        "Rock---Emo",
+        "Rock---Ethereal",
+        "Rock---Experimental",
+        "Rock---Folk Metal",
+        "Rock---Folk Rock",
+        "Rock---Funeral Doom Metal",
+        "Rock---Funk Metal",
+        "Rock---Garage Rock",
+        "Rock---Glam",
+        "Rock---Goregrind",
+        "Rock---Goth Rock",
+        "Rock---Gothic Metal",
+        "Rock---Grindcore",
+        "Rock---Grunge",
+        "Rock---Hard Rock",
+        "Rock---Hardcore",
+        "Rock---Heavy Metal",
+        "Rock---Indie Rock",
+        "Rock---Industrial",
+        "Rock---Krautrock",
+        "Rock---Lo-Fi",
+        "Rock---Lounge",
+        "Rock---Math Rock",
+        "Rock---Melodic Death Metal",
+        "Rock---Melodic Hardcore",
+        "Rock---Metalcore",
+        "Rock---Mod",
+        "Rock---Neofolk",
+        "Rock---New Wave",
+        "Rock---No Wave",
+        "Rock---Noise",
+        "Rock---Noisecore",
+        "Rock---Nu Metal",
+        "Rock---Oi",
+        "Rock---Parody",
+        "Rock---Pop Punk",
+        "Rock---Pop Rock",
+        "Rock---Pornogrind",
+        "Rock---Post Rock",
+        "Rock---Post-Hardcore",
+        "Rock---Post-Metal",
+        "Rock---Post-Punk",
+        "Rock---Power Metal",
+        "Rock---Power Pop",
+        "Rock---Power Violence",
+        "Rock---Prog Rock",
+        "Rock---Progressive Metal",
+        "Rock---Psychedelic Rock",
+        "Rock---Psychobilly",
+        "Rock---Pub Rock",
+        "Rock---Punk",
+        "Rock---Rock & Roll",
+        "Rock---Rockabilly",
+        "Rock---Shoegaze",
+        "Rock---Ska",
+        "Rock---Sludge Metal",
+        "Rock---Soft Rock",
+        "Rock---Southern Rock",
+        "Rock---Space Rock",
+        "Rock---Speed Metal",
+        "Rock---Stoner Rock",
+        "Rock---Surf",
+        "Rock---Symphonic Rock",
+        "Rock---Technical Death Metal",
+        "Rock---Thrash",
+        "Rock---Twist",
+        "Rock---Viking Metal",
+        "Rock---Y\u00e9-Y\u00e9",
+        "Stage & Screen---Musical",
+        "Stage & Screen---Score",
+        "Stage & Screen---Soundtrack",
+        "Stage & Screen---Theme"
+    ],
+    "model_types": [
+        "frozen_model",
+        "SavedModel",
+        "onnx"
+    ],
+    "dataset": {
+        "name": "Discogs-4M (unreleased)",
+        "citation": "In-house dataset",
+        "size": "4M full tracks (3.3M used)",
+        "metrics": {
+            "ROC-AUC": 0.95417,
+            "PR-AUC": 0.20629
+        }
+    },
+    "schema": {
+        "inputs": [
+            {
+                "name": "serving_default_melspectrogram",
+                "type": "float",
+                "shape": [
+                    64,
+                    128,
+                    96
+                ]
+            }
+        ],
+        "outputs": [
+            {
+                "name": "PartitionedCall:0",
+                "type": "float",
+                "shape": [
+                    64,
+                    400
+                ],
+                "op": "Sigmoid",
+                "output_purpose": "predictions"
+            },
+            {
+                "name": "PartitionedCall:1",
+                "type": "float",
+                "shape": [
+                    64,
+                    1280
+                ],
+                "op": "Flatten",
+                "output_purpose": "embeddings"
+            }
+        ]
+    },
+    "citation": "@inproceedings{alonso2022music,\n  title={Music Representation Learning Based on Editorial Metadata from Discogs},\n  author={Alonso-Jim{\\'e}nez, Pablo and Serra, Xavier and Bogdanov, Dmitry},\n  booktitle={Conference of the International Society for Music Information Retrieval (ISMIR)},\n  year={2022}\n}",
+    "inference": {
+        "sample_rate": 16000,
+        "algorithm": "TensorflowPredictEffnetDiscogs"
+    }
+}

scripts/classificacao/genre_discogs400.json

@@ -0,0 +1,452 @@
+{
+  "name": "Genre Discogs400",
+  "type": "Music genre classification",
+  "link": "https://essentia.upf.edu/models/classification-heads/genre_discogs400/genre_discogs400-discogs-effnet-1.pb",
+  "version": "1",
+  "description": "Prediction of 400 music styles in the from the Discogs taxonomy",
+  "author": "Pablo Alonso",
+  "email": "pablo.alonso@upf.edu",
+  "release_date": "2023-05-04",
+  "framework": "tensorflow",
+  "framework_version": "2.8.0",
+  "classes": [
+    "Blues---Boogie Woogie",
+    "Blues---Chicago Blues",
+    "Blues---Country Blues",
+    "Blues---Delta Blues",
+    "Blues---Electric Blues",
+    "Blues---Harmonica Blues",
+    "Blues---Jump Blues",
+    "Blues---Louisiana Blues",
+    "Blues---Modern Electric Blues",
+    "Blues---Piano Blues",
+    "Blues---Rhythm & Blues",
+    "Blues---Texas Blues",
+    "Brass & Military---Brass Band",
+    "Brass & Military---Marches",
+    "Brass & Military---Military",
+    "Children's---Educational",
+    "Children's---Nursery Rhymes",
+    "Children's---Story",
+    "Classical---Baroque",
+    "Classical---Choral",
+    "Classical---Classical",
+    "Classical---Contemporary",
+    "Classical---Impressionist",
+    "Classical---Medieval",
+    "Classical---Modern",
+    "Classical---Neo-Classical",
+    "Classical---Neo-Romantic",
+    "Classical---Opera",
+    "Classical---Post-Modern",
+    "Classical---Renaissance",
+    "Classical---Romantic",
+    "Electronic---Abstract",
+    "Electronic---Acid",
+    "Electronic---Acid House",
+    "Electronic---Acid Jazz",
+    "Electronic---Ambient",
+    "Electronic---Bassline",
+    "Electronic---Beatdown",
+    "Electronic---Berlin-School",
+    "Electronic---Big Beat",
+    "Electronic---Bleep",
+    "Electronic---Breakbeat",
+    "Electronic---Breakcore",
+    "Electronic---Breaks",
+    "Electronic---Broken Beat",
+    "Electronic---Chillwave",
+    "Electronic---Chiptune",
+    "Electronic---Dance-pop",
+    "Electronic---Dark Ambient",
+    "Electronic---Darkwave",
+    "Electronic---Deep House",
+    "Electronic---Deep Techno",
+    "Electronic---Disco",
+    "Electronic---Disco Polo",
+    "Electronic---Donk",
+    "Electronic---Downtempo",
+    "Electronic---Drone",
+    "Electronic---Drum n Bass",
+    "Electronic---Dub",
+    "Electronic---Dub Techno",
+    "Electronic---Dubstep",
+    "Electronic---Dungeon Synth",
+    "Electronic---EBM",
+    "Electronic---Electro",
+    "Electronic---Electro House",
+    "Electronic---Electroclash",
+    "Electronic---Euro House",
+    "Electronic---Euro-Disco",
+    "Electronic---Eurobeat",
+    "Electronic---Eurodance",
+    "Electronic---Experimental",
+    "Electronic---Freestyle",
+    "Electronic---Future Jazz",
+    "Electronic---Gabber",
+    "Electronic---Garage House",
+    "Electronic---Ghetto",
+    "Electronic---Ghetto House",
+    "Electronic---Glitch",
+    "Electronic---Goa Trance",
+    "Electronic---Grime",
+    "Electronic---Halftime",
+    "Electronic---Hands Up",
+    "Electronic---Happy Hardcore",
+    "Electronic---Hard House",
+    "Electronic---Hard Techno",
+    "Electronic---Hard Trance",
+    "Electronic---Hardcore",
+    "Electronic---Hardstyle",
+    "Electronic---Hi NRG",
+    "Electronic---Hip Hop",
+    "Electronic---Hip-House",
+    "Electronic---House",
+    "Electronic---IDM",
+    "Electronic---Illbient",
+    "Electronic---Industrial",
+    "Electronic---Italo House",
+    "Electronic---Italo-Disco",
+    "Electronic---Italodance",
+    "Electronic---Jazzdance",
+    "Electronic---Juke",
+    "Electronic---Jumpstyle",
+    "Electronic---Jungle",
+    "Electronic---Latin",
+    "Electronic---Leftfield",
+    "Electronic---Makina",
+    "Electronic---Minimal",
+    "Electronic---Minimal Techno",
+    "Electronic---Modern Classical",
+    "Electronic---Musique Concr\u00e8te",
+    "Electronic---Neofolk",
+    "Electronic---New Age",
+    "Electronic---New Beat",
+    "Electronic---New Wave",
+    "Electronic---Noise",
+    "Electronic---Nu-Disco",
+    "Electronic---Power Electronics",
+    "Electronic---Progressive Breaks",
+    "Electronic---Progressive House",
+    "Electronic---Progressive Trance",
+    "Electronic---Psy-Trance",
+    "Electronic---Rhythmic Noise",
+    "Electronic---Schranz",
+    "Electronic---Sound Collage",
+    "Electronic---Speed Garage",
+    "Electronic---Speedcore",
+    "Electronic---Synth-pop",
+    "Electronic---Synthwave",
+    "Electronic---Tech House",
+    "Electronic---Tech Trance",
+    "Electronic---Techno",
+    "Electronic---Trance",
+    "Electronic---Tribal",
+    "Electronic---Tribal House",
+    "Electronic---Trip Hop",
+    "Electronic---Tropical House",
+    "Electronic---UK Garage",
+    "Electronic---Vaporwave",
+    "Folk, World, & Country---African",
+    "Folk, World, & Country---Bluegrass",
+    "Folk, World, & Country---Cajun",
+    "Folk, World, & Country---Canzone Napoletana",
+    "Folk, World, & Country---Catalan Music",
+    "Folk, World, & Country---Celtic",
+    "Folk, World, & Country---Country",
+    "Folk, World, & Country---Fado",
+    "Folk, World, & Country---Flamenco",
+    "Folk, World, & Country---Folk",
+    "Folk, World, & Country---Gospel",
+    "Folk, World, & Country---Highlife",
+    "Folk, World, & Country---Hillbilly",
+    "Folk, World, & Country---Hindustani",
+    "Folk, World, & Country---Honky Tonk",
+    "Folk, World, & Country---Indian Classical",
+    "Folk, World, & Country---La\u00efk\u00f3",
+    "Folk, World, & Country---Nordic",
+    "Folk, World, & Country---Pacific",
+    "Folk, World, & Country---Polka",
+    "Folk, World, & Country---Ra\u00ef",
+    "Folk, World, & Country---Romani",
+    "Folk, World, & Country---Soukous",
+    "Folk, World, & Country---S\u00e9ga",
+    "Folk, World, & Country---Volksmusik",
+    "Folk, World, & Country---Zouk",
+    "Folk, World, & Country---\u00c9ntekhno",
+    "Funk / Soul---Afrobeat",
+    "Funk / Soul---Boogie",
+    "Funk / Soul---Contemporary R&B",
+    "Funk / Soul---Disco",
+    "Funk / Soul---Free Funk",
+    "Funk / Soul---Funk",
+    "Funk / Soul---Gospel",
+    "Funk / Soul---Neo Soul",
+    "Funk / Soul---New Jack Swing",
+    "Funk / Soul---P.Funk",
+    "Funk / Soul---Psychedelic",
+    "Funk / Soul---Rhythm & Blues",
+    "Funk / Soul---Soul",
+    "Funk / Soul---Swingbeat",
+    "Funk / Soul---UK Street Soul",
+    "Hip Hop---Bass Music",
+    "Hip Hop---Boom Bap",
+    "Hip Hop---Bounce",
+    "Hip Hop---Britcore",
+    "Hip Hop---Cloud Rap",
+    "Hip Hop---Conscious",
+    "Hip Hop---Crunk",
+    "Hip Hop---Cut-up/DJ",
+    "Hip Hop---DJ Battle Tool",
+    "Hip Hop---Electro",
+    "Hip Hop---G-Funk",
+    "Hip Hop---Gangsta",
+    "Hip Hop---Grime",
+    "Hip Hop---Hardcore Hip-Hop",
+    "Hip Hop---Horrorcore",
+    "Hip Hop---Instrumental",
+    "Hip Hop---Jazzy Hip-Hop",
+    "Hip Hop---Miami Bass",
+    "Hip Hop---Pop Rap",
+    "Hip Hop---Ragga HipHop",
+    "Hip Hop---RnB/Swing",
+    "Hip Hop---Screw",
+    "Hip Hop---Thug Rap",
+    "Hip Hop---Trap",
+    "Hip Hop---Trip Hop",
+    "Hip Hop---Turntablism",
+    "Jazz---Afro-Cuban Jazz",
+    "Jazz---Afrobeat",
+    "Jazz---Avant-garde Jazz",
+    "Jazz---Big Band",
+    "Jazz---Bop",
+    "Jazz---Bossa Nova",
+    "Jazz---Contemporary Jazz",
+    "Jazz---Cool Jazz",
+    "Jazz---Dixieland",
+    "Jazz---Easy Listening",
+    "Jazz---Free Improvisation",
+    "Jazz---Free Jazz",
+    "Jazz---Fusion",
+    "Jazz---Gypsy Jazz",
+    "Jazz---Hard Bop",
+    "Jazz---Jazz-Funk",
+    "Jazz---Jazz-Rock",
+    "Jazz---Latin Jazz",
+    "Jazz---Modal",
+    "Jazz---Post Bop",
+    "Jazz---Ragtime",
+    "Jazz---Smooth Jazz",
+    "Jazz---Soul-Jazz",
+    "Jazz---Space-Age",
+    "Jazz---Swing",
+    "Latin---Afro-Cuban",
+    "Latin---Bai\u00e3o",
+    "Latin---Batucada",
+    "Latin---Beguine",
+    "Latin---Bolero",
+    "Latin---Boogaloo",
+    "Latin---Bossanova",
+    "Latin---Cha-Cha",
+    "Latin---Charanga",
+    "Latin---Compas",
+    "Latin---Cubano",
+    "Latin---Cumbia",
+    "Latin---Descarga",
+    "Latin---Forr\u00f3",
+    "Latin---Guaguanc\u00f3",
+    "Latin---Guajira",
+    "Latin---Guaracha",
+    "Latin---MPB",
+    "Latin---Mambo",
+    "Latin---Mariachi",
+    "Latin---Merengue",
+    "Latin---Norte\u00f1o",
+    "Latin---Nueva Cancion",
+    "Latin---Pachanga",
+    "Latin---Porro",
+    "Latin---Ranchera",
+    "Latin---Reggaeton",
+    "Latin---Rumba",
+    "Latin---Salsa",
+    "Latin---Samba",
+    "Latin---Son",
+    "Latin---Son Montuno",
+    "Latin---Tango",
+    "Latin---Tejano",
+    "Latin---Vallenato",
+    "Non-Music---Audiobook",
+    "Non-Music---Comedy",
+    "Non-Music---Dialogue",
+    "Non-Music---Education",
+    "Non-Music---Field Recording",
+    "Non-Music---Interview",
+    "Non-Music---Monolog",
+    "Non-Music---Poetry",
+    "Non-Music---Political",
+    "Non-Music---Promotional",
+    "Non-Music---Radioplay",
+    "Non-Music---Religious",
+    "Non-Music---Spoken Word",
+    "Pop---Ballad",
+    "Pop---Bollywood",
+    "Pop---Bubblegum",
+    "Pop---Chanson",
+    "Pop---City Pop",
+    "Pop---Europop",
+    "Pop---Indie Pop",
+    "Pop---J-pop",
+    "Pop---K-pop",
+    "Pop---Kay\u014dkyoku",
+    "Pop---Light Music",
+    "Pop---Music Hall",
+    "Pop---Novelty",
+    "Pop---Parody",
+    "Pop---Schlager",
+    "Pop---Vocal",
+    "Reggae---Calypso",
+    "Reggae---Dancehall",
+    "Reggae---Dub",
+    "Reggae---Lovers Rock",
+    "Reggae---Ragga",
+    "Reggae---Reggae",
+    "Reggae---Reggae-Pop",
+    "Reggae---Rocksteady",
+    "Reggae---Roots Reggae",
+    "Reggae---Ska",
+    "Reggae---Soca",
+    "Rock---AOR",
+    "Rock---Acid Rock",
+    "Rock---Acoustic",
+    "Rock---Alternative Rock",
+    "Rock---Arena Rock",
+    "Rock---Art Rock",
+    "Rock---Atmospheric Black Metal",
+    "Rock---Avantgarde",
+    "Rock---Beat",
+    "Rock---Black Metal",
+    "Rock---Blues Rock",
+    "Rock---Brit Pop",
+    "Rock---Classic Rock",
+    "Rock---Coldwave",
+    "Rock---Country Rock",
+    "Rock---Crust",
+    "Rock---Death Metal",
+    "Rock---Deathcore",
+    "Rock---Deathrock",
+    "Rock---Depressive Black Metal",
+    "Rock---Doo Wop",
+    "Rock---Doom Metal",
+    "Rock---Dream Pop",
+    "Rock---Emo",
+    "Rock---Ethereal",
+    "Rock---Experimental",
+    "Rock---Folk Metal",
+    "Rock---Folk Rock",
+    "Rock---Funeral Doom Metal",
+    "Rock---Funk Metal",
+    "Rock---Garage Rock",
+    "Rock---Glam",
+    "Rock---Goregrind",
+    "Rock---Goth Rock",
+    "Rock---Gothic Metal",
+    "Rock---Grindcore",
+    "Rock---Grunge",
+    "Rock---Hard Rock",
+    "Rock---Hardcore",
+    "Rock---Heavy Metal",
+    "Rock---Indie Rock",
+    "Rock---Industrial",
+    "Rock---Krautrock",
+    "Rock---Lo-Fi",
+    "Rock---Lounge",
+    "Rock---Math Rock",
+    "Rock---Melodic Death Metal",
+    "Rock---Melodic Hardcore",
+    "Rock---Metalcore",
+    "Rock---Mod",
+    "Rock---Neofolk",
+    "Rock---New Wave",
+    "Rock---No Wave",
+    "Rock---Noise",
+    "Rock---Noisecore",
+    "Rock---Nu Metal",
+    "Rock---Oi",
+    "Rock---Parody",
+    "Rock---Pop Punk",
+    "Rock---Pop Rock",
+    "Rock---Pornogrind",
+    "Rock---Post Rock",
+    "Rock---Post-Hardcore",
+    "Rock---Post-Metal",
+    "Rock---Post-Punk",
+    "Rock---Power Metal",
+    "Rock---Power Pop",
+    "Rock---Power Violence",
+    "Rock---Prog Rock",
+    "Rock---Progressive Metal",
+    "Rock---Psychedelic Rock",
+    "Rock---Psychobilly",
+    "Rock---Pub Rock",
+    "Rock---Punk",
+    "Rock---Rock & Roll",
+    "Rock---Rockabilly",
+    "Rock---Shoegaze",
+    "Rock---Ska",
+    "Rock---Sludge Metal",
+    "Rock---Soft Rock",
+    "Rock---Southern Rock",
+    "Rock---Space Rock",
+    "Rock---Speed Metal",
+    "Rock---Stoner Rock",
+    "Rock---Surf",
+    "Rock---Symphonic Rock",
+    "Rock---Technical Death Metal",
+    "Rock---Thrash",
+    "Rock---Twist",
+    "Rock---Viking Metal",
+    "Rock---Y\u00e9-Y\u00e9",
+    "Stage & Screen---Musical",
+    "Stage & Screen---Score",
+    "Stage & Screen---Soundtrack",
+    "Stage & Screen---Theme"
+  ],
+  "model_types": ["frozen_model", "SavedModel", "onnx"],
+  "dataset": {
+    "name": "Discogs-4M (unreleased)",
+    "citation": "In-house dataset",
+    "size": "4M full tracks (3.3M used)",
+    "metrics": {
+      "ROC-AUC": 0.95417,
+      "PR-AUC": 0.20629
+    }
+  },
+  "schema": {
+    "inputs": [
+      {
+        "name": "serving_default_model_Placeholder",
+        "type": "float",
+        "shape": ["batch_size", 1280]
+      }
+    ],
+    "outputs": [
+      {
+        "name": "PartitionedCall:0",
+        "type": "float",
+        "shape": ["batch_size", 400],
+        "op": "Sigmoid",
+        "output_purpose": "predictions"
+      }
+    ]
+  },
+  "citation": "@inproceedings{alonso2022music,\n  title={Music Representation Learning Based on Editorial Metadata from Discogs},\n  author={Alonso-Jim{\\'e}nez, Pablo and Serra, Xavier and Bogdanov, Dmitry},\n  booktitle={Conference of the International Society for Music Information Retrieval (ISMIR)},\n  year={2022}\n}",
+  "inference": {
+    "sample_rate": 16000,
+    "algorithm": "TensorflowPredict2D",
+    "embedding_model": {
+      "algorithm": "TensorflowPredictEffnetDiscogs",
+      "model_name": "discogs-effnet-bs64-1",
+      "link": "https://essentia.upf.edu/models/music-style-classification/discogs-effnet/discogs-effnet-bs64-1.pb"
+    }
+  }
+}

scripts/handler/benchmark.py

@@ -0,0 +1,339 @@
+# benchmark.py
+
+import os
+import shutil
+import subprocess
+import multiprocessing
+import logging
+import time
+import platform
+import sys
+import re
+import unicodedata
+import csv
+import psutil
+from datetime import datetime, timedelta
+
+# Importando apenas o necessário (sem sobrescrever as pastas do utils nativo no processamento global)
+from file_parser import parse_mmp_file
+from file_saver import save_to_json, save_to_yaml
+from utils import SRC_MMPSEARCH, MMP_FOLDER
+
+# === ISOLAMENTO DE DIRETÓRIOS PARA O BENCHMARK ===
+# Tudo será gerado dentro de "benchmark_output" para não afetar o Jekyll
+BENCHMARK_OUT = os.path.join(SRC_MMPSEARCH, "benchmark_output")
+DATA_FOLDER = os.path.join(BENCHMARK_OUT, "data")
+METADATA_FOLDER = os.path.join(BENCHMARK_OUT, "metadata")
+WAV_FOLDER = os.path.join(BENCHMARK_OUT, "wav")
+MMPZ_FOLDER = os.path.join(BENCHMARK_OUT, "mmpz")
+LOG_FOLDER = os.path.join(BENCHMARK_OUT, "logs")
+SAIDA_ANALISES = os.path.join(BENCHMARK_OUT, "analises")
+LOTES_FOLDER = os.path.join(BENCHMARK_OUT, "lotes_yaml")
+
+# === CONFIGURAÇÕES GERAIS ===
+TIMEOUT_RENDER_SECONDS = 300
+MIN_RAM_FREE_MB = 500
+
+global_counter = None
+
+
+def slugify(value):
+    value = str(value)
+    value = (
+        unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii")
+    )
+    value = value.lower()
+    value = re.sub(r"[^\w\s-]", "", value)
+    value = re.sub(r"[-\s_]+", "-", value)
+    return value.strip("-_")
+
+
+def get_cpu_safe_count():
+    try:
+        count = multiprocessing.cpu_count()
+        return max(1, count - 1)
+    except:
+        return 1
+
+
+def init_worker(counter):
+    global global_counter
+    global_counter = counter
+
+
+# === FUNÇÃO ORIGINAL DO SEU CÓDIGO (Com leve ajuste de segurança de arquivos) ===
+def process_single_file(args):
+    file_name, clean_slug, total_files = args
+    file_path = os.path.join(MMP_FOLDER, file_name)
+    pid = os.getpid()
+
+    global global_counter
+    current_idx = 0
+    if global_counter:
+        with global_counter.get_lock():
+            global_counter.value += 1
+            current_idx = global_counter.value
+
+    start_time = time.time()
+    result = {
+        "success": False,
+        "file": file_name,
+        "slug": clean_slug,
+        "data": None,
+        "error": None,
+        "duration_s": 0.0,
+        "ram_mb": 0.0,
+        "file_size_mb": 0.0,
+    }
+
+    try:
+        if os.path.exists(file_path):
+            result["file_size_mb"] = os.path.getsize(file_path) / (1024 * 1024)
+
+        mem = psutil.virtual_memory()
+        if (mem.available / (1024 * 1024)) < MIN_RAM_FREE_MB:
+            result["error"] = "RAM Insuficiente"
+            return result
+
+        original_base_name = os.path.splitext(file_name)[0]
+        ogg_name = clean_slug + ".ogg"
+        json_name = clean_slug + ".json"
+        yml_name = clean_slug + ".yml"
+        target_mmp_path = ""
+
+        if file_name.endswith(".mmpz"):
+            destination_path = os.path.join(MMPZ_FOLDER, file_name)
+            # ALERTA DE BENCHMARK: Trocado move por copy2 para não destruir o dataset de testes!
+            if not os.path.exists(destination_path):
+                shutil.copy2(file_path, destination_path)
+
+            mmp_temp_name = clean_slug + ".mmp"
+            output_mmp_path = os.path.join(MMP_FOLDER, mmp_temp_name)
+            abs_dest = os.path.abspath(destination_path)
+            abs_mmp_out = os.path.abspath(output_mmp_path)
+
+            with open(abs_mmp_out, "w") as outfile:
+                subprocess.run(
+                    ["lmms", "--dump", abs_dest],
+                    stdout=outfile,
+                    stderr=subprocess.PIPE,
+                    check=True,
+                    timeout=60,
+                    env={"QT_QPA_PLATFORM": "offscreen", **os.environ},
+                )
+            target_mmp_path = output_mmp_path
+        elif file_name.endswith(".mmp"):
+            target_mmp_path = file_path
+
+        # Renderização do Áudio
+        if os.path.exists(target_mmp_path):
+            abs_target_mmp = os.path.abspath(target_mmp_path)
+            abs_ogg_out = os.path.abspath(os.path.join(WAV_FOLDER, ogg_name))
+
+            # Aqui estava o errinho do "wav_cmd" que arrumamos na revisão!
+            ogg_cmd = ["lmms", "-r", abs_target_mmp, "-o", abs_ogg_out, "-f", "ogg"]
+            render_process = subprocess.run(
+                ogg_cmd,
+                check=False,
+                capture_output=True,
+                text=True,
+                timeout=TIMEOUT_RENDER_SECONDS,
+                env={"QT_QPA_PLATFORM": "offscreen", **os.environ},
+            )
+            if render_process.returncode != 0:
+                raise subprocess.CalledProcessError(
+                    render_process.returncode, "lmms render", stderr="Erro LMMS"
+                )
+
+        # Parsing do XML
+        if os.path.exists(target_mmp_path):
+            mmp_data = parse_mmp_file(target_mmp_path)
+            if mmp_data:
+                mmp_data["file"] = clean_slug
+                mmp_data["original_title"] = original_base_name
+                save_to_json(mmp_data, os.path.join(METADATA_FOLDER, json_name))
+                save_to_yaml(mmp_data, os.path.join(DATA_FOLDER, yml_name))
+                result["success"] = True
+                result["data"] = mmp_data
+        
+        try:
+            process = psutil.Process(pid)
+            # rss = Resident Set Size (RAM real usada pelo processo)
+            result["ram_mb"] = process.memory_info().rss / (1024 * 1024)
+        except:
+            result["ram_mb"] = 0.0
+
+    except Exception as e:
+        result["error"] = str(e)
+
+    result["duration_s"] = time.time() - start_time
+    return result
+
+
+# === MOTOR DE BENCHMARK ===
+def execute_batch(limit_files, mode):
+    """
+    Executa um lote de arquivos.
+    Se mode == 'sequencial', usa apenas 1 núcleo.
+    Se mode == 'paralelo', usa o safe count de núcleos.
+    """
+    start_time_global = time.time()
+
+    # Prepara pastas limpas para o teste
+    for folder in [
+        MMPZ_FOLDER,
+        WAV_FOLDER,
+        METADATA_FOLDER,
+        DATA_FOLDER,
+        LOTES_FOLDER,
+        SAIDA_ANALISES,
+    ]:
+        os.makedirs(folder, exist_ok=True)
+
+    all_files_raw = [f for f in os.listdir(MMP_FOLDER) if f.endswith((".mmp", ".mmpz"))]
+    if limit_files > 0:
+        all_files_raw = all_files_raw[:limit_files]
+
+    total_files = len(all_files_raw)
+    if total_files == 0:
+        return 0, 0, 0, 0, 0
+
+    shared_counter = multiprocessing.Value("i", 0)
+    tasks = []
+
+    for idx, file_name in enumerate(all_files_raw):
+        slug = slugify(os.path.splitext(file_name)[0])
+        if not slug:
+            slug = f"proj-{idx}"
+        tasks.append((file_name, slug, total_files))
+
+    num_cores = 1 if mode == "sequencial" else get_cpu_safe_count()
+
+    print(
+        f"\n[{mode.upper()}] Processando {total_files} arquivos com {num_cores} núcleo(s)..."
+    )
+
+    # Execução
+    with multiprocessing.Pool(
+        processes=num_cores, initializer=init_worker, initargs=(shared_counter,)
+    ) as pool:
+        results = pool.map(process_single_file, tasks)
+
+    # --- CÁLCULO DE MÉDIAS E SUCESSOS ---
+    sucessos = [r for r in results if r["success"]]
+    qtd_sucessos = len(sucessos)
+
+    avg_time = (
+        sum(r["duration_s"] for r in sucessos) / qtd_sucessos if qtd_sucessos > 0 else 0
+    )
+    avg_size = (
+        sum(r["file_size_mb"] for r in sucessos) / qtd_sucessos
+        if qtd_sucessos > 0
+        else 0
+    )
+    avg_ram = (
+        sum(r["ram_mb"] for r in sucessos) / qtd_sucessos if qtd_sucessos > 0 else 0
+    )
+
+    duration_total = time.time() - start_time_global
+
+    # --- SALVAR RELATÓRIO DETALHADO POR LOTE ---
+    detalhado_csv = os.path.join(
+        BENCHMARK_OUT, f"auditoria_detalhada_{mode}_{limit_files}.csv"
+    )
+    with open(detalhado_csv, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(
+            ["Arquivo", "Tamanho_MB", "Tempo_s", "RAM_MB", "Status", "Erro"]
+        )
+        for r in results:
+            status_str = "SUCESSO" if r["success"] else "FALHA"
+            writer.writerow(
+                [
+                    r["file"],
+                    f"{r['file_size_mb']:.2f}",
+                    f"{r['duration_s']:.2f}",
+                    f"{r['ram_mb']:.2f}",
+                    status_str,
+                    r["error"] or "",
+                ]
+            )
+
+    print(
+        f"[{mode.upper()}] Concluído em {duration_total:.2f}s (Sucessos: {qtd_sucessos}/{total_files})"
+    )
+    print(
+        f"[{mode.upper()}] Médias -> Tempo: {avg_time:.2f}s | Tamanho: {avg_size:.2f}MB | RAM: {avg_ram:.2f}MB"
+    )
+
+    return duration_total, qtd_sucessos, avg_time, avg_size, avg_ram
+
+
+def run_benchmark():
+    print("==================================================")
+    print(" INICIANDO TESTE DE SPEEDUP (SEQUENCIAL VS PARALELO)")
+    print("==================================================")
+
+    test_sizes = [1, 10, 100, 1000]
+    modes = ["sequencial", "paralelo"]
+
+    results_table = []
+
+    for size in test_sizes:
+        print(f"\n>>> INICIANDO BATERIA PARA {size} ARQUIVO(S) <<<")
+        row = {"Tamanho": size}
+
+        for mode in modes:
+            duration, success_count, avg_time, avg_size, avg_ram = execute_batch(
+                size, mode
+            )
+            row[f"Tempo_Total_{mode}_(s)"] = round(duration, 2)
+            row[f"Tempo_Medio_Proj_{mode}_(s)"] = round(avg_time, 2)
+            row[f"Tamanho_Medio_Proj_(MB)"] = round(
+                avg_size, 2
+            )  # O tamanho é igual para ambos, mas guardamos
+            row[f"RAM_Media_{mode}_(MB)"] = round(avg_ram, 2)
+            row[f"Sucesso_{mode}"] = success_count
+
+        # Calcula o Speedup (Tempo Total Sequencial / Tempo Total Paralelo)
+        speedup = (
+            row["Tempo_Total_sequencial_(s)"] / row["Tempo_Total_paralelo_(s)"]
+            if row["Tempo_Total_paralelo_(s)"] > 0
+            else 0
+        )
+        row["Speedup"] = round(speedup, 2)
+        results_table.append(row)
+
+        print(f"--- Fim da bateria de {size}. Speedup Alcançado: {speedup:.2f}x ---")
+
+    # Salva o resultado final num CSV consolidado
+    os.makedirs(BENCHMARK_OUT, exist_ok=True)
+    csv_file = os.path.join(BENCHMARK_OUT, "resultado_grafico_speedup.csv")
+
+    # Lista de colunas para o CSV final
+    colunas = [
+        "Tamanho",
+        "Speedup",
+        "Tempo_Total_sequencial_(s)",
+        "Tempo_Total_paralelo_(s)",
+        "Tempo_Medio_Proj_sequencial_(s)",
+        "Tempo_Medio_Proj_paralelo_(s)",
+        "RAM_Media_sequencial_(MB)",
+        "RAM_Media_paralelo_(MB)",
+        "Tamanho_Medio_Proj_(MB)",
+        "Sucesso_sequencial",
+        "Sucesso_paralelo",
+    ]
+
+    with open(csv_file, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=colunas)
+        writer.writeheader()
+        writer.writerows(results_table)
+
+    print("\n==================================================")
+    print(f"BENCHMARK FINALIZADO! Resultados salvos em:\n{BENCHMARK_OUT}")
+    print("==================================================")
+
+
+if __name__ == "__main__":
+    run_benchmark()

scripts/handler/benchmark_output.log

@@ -0,0 +1,53 @@
+nohup: ignoring input
+==================================================
+ INICIANDO TESTE DE SPEEDUP (SEQUENCIAL VS PARALELO)
+==================================================
+
+>>> INICIANDO BATERIA PARA 1 ARQUIVO(S) <<<
+
+[SEQUENCIAL] Processando 1 arquivos com 1 núcleo(s)...
+[SEQUENCIAL] Concluído em 43.78s (Sucessos: 1/1)
+[SEQUENCIAL] Médias -> Tempo: 43.69s | Tamanho: 0.54MB | RAM: 27.22MB
+
+[PARALELO] Processando 1 arquivos com 11 núcleo(s)...
+[PARALELO] Concluído em 46.51s (Sucessos: 1/1)
+[PARALELO] Médias -> Tempo: 46.48s | Tamanho: 0.54MB | RAM: 27.58MB
+--- Fim da bateria de 1. Speedup Alcançado: 0.94x ---
+
+>>> INICIANDO BATERIA PARA 10 ARQUIVO(S) <<<
+
+[SEQUENCIAL] Processando 10 arquivos com 1 núcleo(s)...
+Process ForkPoolWorker-13:
+Traceback (most recent call last):
+  File "/usr/lib/python3.12/multiprocessing/pool.py", line 131, in worker
+    put((job, i, result))
+  File "/usr/lib/python3.12/multiprocessing/queues.py", line 399, in put
+    self._writer.send_bytes(obj)
+  File "/usr/lib/python3.12/multiprocessing/connection.py", line 200, in send_bytes
+    self._send_bytes(m[offset:offset + size])
+  File "/usr/lib/python3.12/multiprocessing/connection.py", line 420, in _send_bytes
+    self._send(header)
+  File "/usr/lib/python3.12/multiprocessing/connection.py", line 384, in _send
+    n = write(self._handle, buf)
+        ^^^^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/usr/lib/python3.12/multiprocessing/pool.py", line 136, in worker
+    put((job, i, (False, wrapped)))
+  File "/usr/lib/python3.12/multiprocessing/queues.py", line 399, in put
+    self._writer.send_bytes(obj)
+  File "/usr/lib/python3.12/multiprocessing/connection.py", line 200, in send_bytes
+    self._send_bytes(m[offset:offset + size])
+  File "/usr/lib/python3.12/multiprocessing/connection.py", line 420, in _send_bytes
+    self._send(header)
+  File "/usr/lib/python3.12/multiprocessing/connection.py", line 384, in _send
+    n = write(self._handle, buf)
+        ^^^^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe

scripts/handler/crawler.py

@@ -0,0 +1,230 @@
+import os
+import time
+import threading
+import requests
+import csv
+import psutil
+import re
+import unicodedata
+from datetime import datetime, timedelta
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse, parse_qs
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from utils import (
+    SRC_MMPSEARCH,
+    MMP_FOLDER,
+)
+
+# ================= CONFIGURAÇÕES PADRONIZADAS =================
+BASE_DIR = SRC_MMPSEARCH
+PASTA_DESTINO = MMP_FOLDER # Pasta de entrada do pipeline
+LOG_DIR = os.path.join(BASE_DIR, "logs/crawler")
+ARQUIVO_AUDITORIA = os.path.join(LOG_DIR, "audit_crawler.csv")
+
+# Segurança de Hardware
+MIN_RAM_FREE_MB = 500  # Pausa se tiver menos que 500MB livre
+MAX_WORKERS_SAFE = 4   # I/O Bound não precisa de tantos workers quanto CPU, mas 4 é seguro
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Compatible; MMPResearchBot/1.0; +http://seu-site.com)'
+}
+
+# Cria pastas
+for p in [PASTA_DESTINO, LOG_DIR]:
+    os.makedirs(p, exist_ok=True)
+
+# ================= FUNÇÕES AUXILIARES =================
+
+def slugify(value):
+    """Sanitização robusta de nomes de arquivo (Padrão do Pipeline)"""
+    value = str(value)
+    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
+    value = value.lower()
+    value = re.sub(r'[^\w\s-]', '', value)
+    value = re.sub(r'[-\s_]+', '-', value)
+    return value.strip('-_')
+
+def verificar_memoria_segura():
+    """Retorna True se é seguro continuar"""
+    try:
+        mem = psutil.virtual_memory()
+        return (mem.available / (1024 * 1024)) > MIN_RAM_FREE_MB
+    except:
+        return True
+
+# ================= CLASSE DE AUDITORIA =================
+class AuditoriaCrawler:
+    def __init__(self, arquivo_csv):
+        self.arquivo_csv = arquivo_csv
+        self.inicio_global = time.time()
+        self.lock = threading.Lock()
+        
+        # Cria CSV se não existir
+        if not os.path.exists(self.arquivo_csv):
+            with open(self.arquivo_csv, 'w', newline='', encoding='utf-8') as f:
+                writer = csv.writer(f)
+                writer.writerow([
+                    "Timestamp", "Nome_Original", "Slug_Final", 
+                    "Tamanho_MB", "Tempo_DL_s", "Velocidade_KBps", 
+                    "Status", "URL_Origem"
+                ])
+
+    def registrar_download(self, nome_orig, slug, tamanho_bytes, tempo_s, url, status="SUCESSO"):
+        with self.lock:
+            try:
+                tamanho_mb = tamanho_bytes / (1024 * 1024)
+                velocidade = (tamanho_bytes / 1024) / tempo_s if tempo_s > 0 else 0
+                ts = datetime.now().strftime("%H:%M:%S")
+                
+                with open(self.arquivo_csv, 'a', newline='', encoding='utf-8') as f:
+                    writer = csv.writer(f)
+                    writer.writerow([
+                        ts, nome_orig, slug, 
+                        f"{tamanho_mb:.2f}", f"{tempo_s:.2f}", 
+                        f"{velocidade:.2f}", status, url
+                    ])
+                
+                # Log Console Simplificado
+                print(f"[{ts}] {status}: {slug} ({tamanho_mb:.2f}MB @ {velocidade:.0f}KB/s)")
+            except Exception as e:
+                print(f"Erro ao auditar: {e}")
+
+# ================= WORKER =================
+
+def baixar_projeto_worker(args):
+    url_detalhes, auditor = args
+    
+    # Check de Segurança (Throttle)
+    if not verificar_memoria_segura():
+        time.sleep(5) # Espera RAM liberar
+        if not verificar_memoria_segura():
+            return (False, "RAM Crítica - Download Abortado")
+
+    try:
+        # 1. Pega página de detalhes
+        resp = requests.get(url_detalhes, headers=HEADERS, timeout=15)
+        if resp.status_code != 200:
+            auditor.registrar_download("N/A", "N/A", 0, 0, url_detalhes, f"HTTP_{resp.status_code}")
+            return (False, f"HTTP {resp.status_code}")
+
+        soup = BeautifulSoup(resp.text, 'html.parser')
+        
+        # Lógica de encontrar link de download (Manteve a sua, que é boa)
+        link_tag = soup.find('a', href=lambda h: h and 'download_file.php' in h)
+        if not link_tag:
+            link_tag = soup.find('a', string='Download')
+
+        if link_tag:
+            url_download = urljoin(url_detalhes, link_tag['href'])
+            
+            inicio_dl = time.time()
+            
+            # Request do Arquivo
+            with requests.get(url_download, headers=HEADERS, stream=True, timeout=60) as r_file:
+                r_file.raise_for_status()
+                
+                # --- DESCIFRAR NOME (COM SANITIZAÇÃO) ---
+                nome_bruto = ""
+                if "Content-Disposition" in r_file.headers:
+                    cd = r_file.headers["Content-Disposition"]
+                    if "filename=" in cd:
+                        nome_bruto = cd.split("filename=")[1].strip('"')
+                
+                if not nome_bruto:
+                    query = parse_qs(urlparse(url_download).query)
+                    nome_bruto = query['name'][0] if 'name' in query else f"proj_{int(time.time())}.mmpz"
+                
+                # Aplica Slugify (Padrão do Pipeline)
+                nome_base, ext = os.path.splitext(nome_bruto)
+                if not ext: ext = ".mmpz" # Default seguro
+                slug = slugify(nome_base)
+                nome_final = f"{slug}{ext}"
+                
+                caminho_final = os.path.join(PASTA_DESTINO, nome_final)
+
+                # Skip se já existe
+                if os.path.exists(caminho_final):
+                    auditor.registrar_download(nome_bruto, slug, os.path.getsize(caminho_final), 0, url_detalhes, "SKIPPED_EXISTS")
+                    return (True, "Já existe")
+
+                # Gravação em chunks (Seguro para RAM)
+                bytes_downloaded = 0
+                with open(caminho_final, 'wb') as f:
+                    for chunk in r_file.iter_content(chunk_size=8192):
+                        if chunk:
+                            f.write(chunk)
+                            bytes_downloaded += len(chunk)
+            
+            tempo_total = time.time() - inicio_dl
+            
+            # Auditoria de Sucesso
+            auditor.registrar_download(nome_bruto, slug, bytes_downloaded, tempo_total, url_detalhes, "SUCESSO")
+            return (True, "OK")
+
+        else:
+            auditor.registrar_download("N/A", "N/A", 0, 0, url_detalhes, "LINK_NOT_FOUND")
+            return (False, "Link não encontrado")
+
+    except Exception as e:
+        auditor.registrar_download("Erro", "Erro", 0, 0, url_detalhes, f"EXCEPT_{str(e)[:20]}")
+        return (False, str(e))
+
+# ================= MAIN =================
+
+def crawler_manager(url_base, max_paginas=2):
+    print(f"\n=== INICIANDO CRAWLER PADRONIZADO ===")
+    print(f"Destino: {PASTA_DESTINO}")
+    print(f"Auditoria: {ARQUIVO_AUDITORIA}")
+    
+    auditor = AuditoriaCrawler(ARQUIVO_AUDITORIA)
+    pagina_atual = 1
+    total_sucesso = 0
+    
+    # Executor seguro
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS_SAFE) as executor:
+        while pagina_atual <= max_paginas:
+            print(f"\n--- Varrendo Página {pagina_atual} ---")
+            
+            operador = '&' if '?' in url_base else '?'
+            url_pag = f"{url_base}{operador}page={pagina_atual}"
+            
+            try:
+                # Pega a lista de projetos da página
+                r = requests.get(url_pag, headers=HEADERS)
+                soup = BeautifulSoup(r.text, 'html.parser')
+                # Filtro específico do LMMS Sharing Platform
+                links = soup.find_all('a', href=lambda h: h and '?action=show&file=' in h)
+                urls_unicas = sorted(list(set([l['href'] for l in links])))
+                
+                if not urls_unicas:
+                    print("Nenhum projeto encontrado nesta página. Encerrando.")
+                    break
+
+                # Submete tarefas
+                futures = []
+                for u in urls_unicas:
+                    url_completa = urljoin(url_base, u)
+                    futures.append(executor.submit(baixar_projeto_worker, (url_completa, auditor)))
+
+                # Processa resultados conforme chegam
+                for future in as_completed(futures):
+                    ok, msg = future.result()
+                    if ok: total_sucesso += 1
+
+            except Exception as e:
+                print(f"Erro na paginação: {e}")
+            
+            pagina_atual += 1
+            time.sleep(1) # Respeito ao servidor
+
+    tempo_total = time.time() - auditor.inicio_global
+    print(f"\n=== CRAWLER FINALIZADO ===")
+    print(f"Tempo: {timedelta(seconds=int(tempo_total))}")
+    print(f"Downloads com Sucesso: {total_sucesso}")
+
+if __name__ == "__main__":
+    # URL de exemplo (Projetos ordenados por data)
+    URL_ALVO = 'https://lmms.io/lsp/?action=browse&category=Projects&sort=date'
+    
+    crawler_manager(URL_ALVO, max_paginas=499)

scripts/login/login.py

@@ -0,0 +1,72 @@
+from flask import Flask, request, jsonify, session
+from flask_sqlalchemy import SQLAlchemy
+from flask_bcrypt import Bcrypt
+from flask_login import LoginManager, UserMixin, login_user, login_required, logout_user, current_user
+
+app = Flask(__name__)
+app.config['SECRET_KEY'] = '25de5592bf94c2ca18e27baa0ae2d4ee22a63012f32e1be719d31f530c215a387b9ec0c9d96be38e80a7ccdd859e04408facefff8fd9119e7f5a2d987d85abb7' # Troque isso!
+app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///users_mmpSearch.db' # O arquivo do banco
+db = SQLAlchemy(app)
+bcrypt = Bcrypt(app)
+login_manager = LoginManager(app)
+
+# --- Modelo do Banco de Dados ---
+class User(UserMixin, db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    username = db.Column(db.String(150), unique=True, nullable=False)
+    password = db.Column(db.String(150), nullable=False)
+
+# Cria o banco na primeira execução
+with app.app_context():
+    db.create_all()
+
+@login_manager.user_loader
+def load_user(user_id):
+    return User.query.get(int(user_id))
+
+# --- Rotas ---
+
+@app.route('/api/register', methods=['POST'])
+def register():
+    data = request.json
+    hashed_password = bcrypt.generate_password_hash(data['password']).decode('utf-8')
+    new_user = User(username=data['username'], password=hashed_password)
+    try:
+        db.session.add(new_user)
+        db.session.commit()
+        return jsonify({"message": "Usuário criado com sucesso!"}), 201
+    except:
+        return jsonify({"message": "Usuário já existe"}), 400
+
+@app.route('/api/login', methods=['POST'])
+def login():
+    data = request.json
+    user = User.query.filter_by(username=data['username']).first()
+    if user and bcrypt.check_password_hash(user.password, data['password']):
+        login_user(user)
+        return jsonify({"message": "Login realizado", "user": user.username}), 200
+    return jsonify({"message": "Credenciais inválidas"}), 401
+
+@app.route('/api/logout', methods=['POST'])
+@login_required
+def logout():
+    logout_user()
+    return jsonify({"message": "Logout realizado"}), 200
+
+@app.route('/api/check_auth', methods=['GET'])
+def check_auth():
+    if current_user.is_authenticated:
+        return jsonify({"logged_in": True, "user": current_user.username})
+    return jsonify({"logged_in": False})
+
+# Rota para seus uploads (exemplo de como proteger)
+@app.route('/api/upload_seguro', methods=['POST'])
+@login_required
+def upload_seguro():
+    # Aqui entraria sua lógica de salvar o arquivo
+    # Como você já tem a lógica pronta, você pode chamar ela aqui
+    # ou mover o código para cá.
+    return jsonify({"message": f"Upload recebido de {current_user.username}"})
+
+if __name__ == '__main__':
+    app.run(port=33005)

scripts/pipeline_etl.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Interrompe a execução imediatamente se qualquer comando falhar
+set -e
+
+# Define o caminho absoluto para o seu Python do ambiente virtual
+PYTHON_BIN="/nethome/jotachina/projetos/mmpSearch/venv/bin/python3"
+
+# Define o diretório raiz do projeto para garantir que os caminhos relativos funcionem
+BASE_DIR="/nethome/jotachina/projetos/mmpSearch/scripts"
+
+echo "================================================================="
+echo "Iniciando Pipeline Completo do MMPSearch"
+echo "================================================================="
+
+# Navega até a raiz do projeto para que os caminhos dos scripts funcionem corretamente
+cd "$BASE_DIR"
+
+echo "[1/3] Executando: handler/main.py..."
+"$PYTHON_BIN" handler/main.py
+echo "handler/main.py concluído!"
+echo "-----------------------------------------------------------------"
+
+echo "[2/3] Executando: classificacao/analise_audio.py..."
+"$PYTHON_BIN" classificacao/analise_audio.py
+echo "classificacao/analise_audio.py concluído!"
+echo "-----------------------------------------------------------------"
+
+echo "[3/3] Executando: classificacao/classificacao_mestre.py..."
+"$PYTHON_BIN" classificacao/classificacao_mestre.py
+echo "classificacao/classificacao_mestre.py concluído!"
+echo "================================================================="
+echo "Pipeline finalizado com sucesso!"
+echo "================================================================="