diff --git a/.gitignore b/.gitignore index d20d29d..19775a6 100755 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,5 @@ venv .gitignore .git .gitea -scripts src assets/js/creations/server/data diff --git a/scripts/classificacao/analise_audio.py b/scripts/classificacao/analise_audio.py new file mode 100644 index 0000000..cada2f6 --- /dev/null +++ b/scripts/classificacao/analise_audio.py @@ -0,0 +1,631 @@ +import os +import sys +import multiprocessing +import concurrent.futures # NOVO: Necessário para usar o ProcessPoolExecutor +import csv +import time +import glob +from datetime import datetime, timedelta + +# ================= 1. LIMITADOR DE CPU DINÂMICO ================= +def configurar_limites_cpu(): + try: + total_cores = multiprocessing.cpu_count() + + print("--- Configuração de Hardware ---") + print(f"Núcleos Totais: {total_cores}") + print("Núcleos Alocados: 1 thread por processo (Multiprocessing Real)") + print("Núcleos Reservados para o Sistema: 1") + + # IMPORTANTE: Em processamento paralelo real (ProcessPoolExecutor), + # as libs em C++ devem usar apenas "1" thread para evitar colisão na CPU (oversubscription) + os.environ["OMP_NUM_THREADS"] = "1" + os.environ["MKL_NUM_THREADS"] = "1" + os.environ["TF_NUM_INTRAOP_THREADS"] = "1" + os.environ["TF_NUM_INTEROP_THREADS"] = "1" + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" + + except Exception as e: + print(f"Aviso: Não foi possível limitar CPU automaticamente: {e}") + +configurar_limites_cpu() + +# ================= IMPORTS ================= +import json +import yaml +import logging +import gc +import numpy as np +import psutil +import essentia +import essentia.standard as es +from tqdm import tqdm +import unicodedata + +# Desativa logs internos do Essentia +essentia.log.info.active = False +essentia.log.warning.active = False + +# ================= CONFIGURAÇÕES ================= +BASE_DIR = "/var/www/html/trens/src_mmpSearch" +PASTA_WAVS = os.path.join(BASE_DIR, "wav") +PASTA_LOTES_YAML = os.path.join(BASE_DIR, "lotes_yaml") + +# --- SISTEMA DE DOIS MODELOS --- +MODELO_EMBEDDING = "discogs-effnet-bs64-1.pb" +MODELO_CLASSIFIER = "genre_discogs400.pb" +MODELO_CLASSES = "genre_discogs400.json" + +# Saídas +DIR_SAIDA = os.path.join(BASE_DIR, "saida_analises") +os.makedirs(DIR_SAIDA, exist_ok=True) +ARQUIVO_CHECKPOINT = os.path.join(DIR_SAIDA, "checkpoint_analises.jsonl") +ARQUIVO_FINAL_JSON = os.path.join(DIR_SAIDA, "db_final_completo.json") +ARQUIVO_AUDITORIA = os.path.join(DIR_SAIDA, "audit_performance.csv") + +# Limites de Segurança +LIMITE_RAM_MINIMA_MB = 500 +MAX_DURACAO_ANALISE = 20 * 60 + +# Logs +LOG_DIR = os.path.join(BASE_DIR, "logs/classificacao") +os.makedirs(LOG_DIR, exist_ok=True) +TIMESTAMP = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") +logging.basicConfig( + filename=os.path.join(LOG_DIR, f"log_{TIMESTAMP}.log"), + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) + +DATABASE_YAML = {} + +# Variáveis globais para os Workers conseguirem enxergar os modelos em memória +GLOBAL_EMBEDDING = None +GLOBAL_CLASSIFIER = None +GLOBAL_CLASSES = None + +# ================= CLASSE DE AUDITORIA OTIMIZADA ================= +class Auditoria: + def __init__(self, arquivo_csv): + self.arquivo_csv = arquivo_csv + self.inicio_global = time.time() + self.process = psutil.Process(os.getpid()) + + # Mantém o arquivo aberto em modo 'append' na memória (Resolve Gargalo HD) + cabecalho = not os.path.exists(self.arquivo_csv) + self.file = open(self.arquivo_csv, "a", newline="", encoding="utf-8") + self.writer = csv.writer(self.file) + + if cabecalho: + self.writer.writerow( + [ + "timestamp", + "arquivo", + "tamanho_mb", + "duracao_audio_s", + "tempo_proc_s", + "ram_uso_mb", + "cpu_percent", + ] + ) + self.file.flush() + + def registrar_processamento(self, nome_arquivo, tamanho_bytes, duracao_audio, tempo_gasto): + try: + ram_mb = self.process.memory_info().rss / (1024 * 1024) + cpu_pct = self.process.cpu_percent(interval=None) + tamanho_mb = tamanho_bytes / (1024 * 1024) + agora = datetime.now().strftime("%H:%M:%S") + + # Escreve na mesma conexão de arquivo aberta + self.writer.writerow( + [ + agora, + nome_arquivo, + round(tamanho_mb, 2), + round(duracao_audio, 2), + round(tempo_gasto, 4), + round(ram_mb, 2), + cpu_pct, + ] + ) + self.file.flush() + except Exception as e: + logging.error(f"Erro ao auditar arquivo: {e}") + + def verificar_marco(self, contagem): + pass # Você pode customizar logs de progresso aqui depois se quiser + + def fechar(self): + try: + self.file.close() + except: + pass + + +# ================= FUNÇÕES AUXILIARES ================= + +def verificar_memoria_segura(tamanho_arquivo_bytes): + mem = psutil.virtual_memory() + livre_mb = mem.available / (1024 * 1024) + estimativa_uso_ram = tamanho_arquivo_bytes * 15 + uso_mb = estimativa_uso_ram / (1024 * 1024) + + if livre_mb < LIMITE_RAM_MINIMA_MB: + print(f" [!] RAM Crítica ({livre_mb:.1f}MB livres). Forçando limpeza...") + gc.collect() + time.sleep(2) + mem = psutil.virtual_memory() + if mem.available / (1024 * 1024) < LIMITE_RAM_MINIMA_MB: + return False, "RAM insuficiente no sistema" + + if uso_mb > (livre_mb * 0.8): + return ( + False, + f"Arquivo muito grande para RAM atual (Est: {uso_mb:.1f}MB, Livre: {livre_mb:.1f}MB)", + ) + + return True, "OK" + + +def normalizar_chave(texto): + if not texto: + return "" + s = str(texto) + if "/" in s: + s = os.path.basename(s) + s = os.path.splitext(s)[0] + s = unicodedata.normalize("NFKD", s).encode("ASCII", "ignore").decode("ASCII") + return s.lower().replace(" ", "").replace("-", "").replace("_", "").strip() + + +def carregar_database_yaml(): + print(f"--- Carregando Metadados de Lotes YAML em: {PASTA_LOTES_YAML} ---") + + padrao = os.path.join(PASTA_LOTES_YAML, "*.yml") + arquivos_lote = sorted(glob.glob(padrao)) + + db = {} + + if not arquivos_lote: + logging.warning(f"Nenhum lote encontrado em {PASTA_LOTES_YAML}") + return {} + + total_carregados = 0 + + for arquivo in arquivos_lote: + try: + with open(arquivo, "r", encoding="utf-8") as f: + dados_lote = yaml.safe_load(f) + + if not dados_lote: + continue + + lista = dados_lote if isinstance(dados_lote, list) else [dados_lote] + + for proj in lista: + if not isinstance(proj, dict): + continue + + candidatos = [ + proj.get("file"), + proj.get("original_title"), + proj.get("src"), + ] + for k in candidatos: + if k: + chave = normalizar_chave(k) + if chave: + db[chave] = proj + + total_carregados += len(lista) + print(f"-> Carregado: {os.path.basename(arquivo)} ({len(lista)} projetos)") + + except Exception as e: + logging.error(f"Erro ao ler lote {arquivo}: {e}") + print(f"X Erro ao ler {os.path.basename(arquivo)}") + + print(f"--- Total de Projetos no DB (Memória): {len(db)} (de {total_carregados} lidos) ---") + return db + + +def carregar_checkpoint(): + processados = set() + dados_existentes = [] + if os.path.exists(ARQUIVO_CHECKPOINT): + with open(ARQUIVO_CHECKPOINT, "r", encoding="utf-8") as f: + for linha in f: + try: + obj = json.loads(linha) + reprocessar = False + if "analise_ia" in obj and "erro" in obj["analise_ia"]: + reprocessar = True + if "analise_tecnica" in obj: + est = obj["analise_tecnica"].get("estrutura_segundos", []) + if len(est) == 2 and est[1] == 12.0: + reprocessar = True + if reprocessar: + continue + processados.add(obj["arquivo"]) + dados_existentes.append(obj) + except: + pass + return processados, dados_existentes + + +def salvar_progresso(resultado): + with open(ARQUIVO_CHECKPOINT, "a", encoding="utf-8") as f: + json.dump(resultado, f, ensure_ascii=False) + f.write("\n") + + +def calcular_complexidade(projeto_yaml): + score = 0 + W_INST_SYNTH, W_INST_SAMPLE = 3.0, 1.0 + W_AUTOMATION, W_PATTERN, W_FX = 2.0, 0.5, 1.5 + + stats = {"num_automations": 0, "num_patterns": 0, "num_effects": 0, "num_synths": 0} + + tracks = projeto_yaml.get("tracks", []) + for track in tracks: + instruments = track.get("instruments", []) + for inst in instruments: + nome_plugin = inst.get("plugin_name", "").lower() + nome_inst = inst.get("instrument_name", "").lower() + + if "automation" in nome_inst or "automation" in nome_plugin: + score += W_AUTOMATION + stats["num_automations"] += 1 + elif nome_plugin in [ + "zynaddsubfx", + "monstro", + "lb302", + "watsyn", + "opyd", + "sfxr", + ]: + score += W_INST_SYNTH + stats["num_synths"] += 1 + else: + score += W_INST_SAMPLE + + patterns = inst.get("patterns", []) + qtd_patterns = len(patterns) + score += qtd_patterns * W_PATTERN + stats["num_patterns"] += qtd_patterns + + fxchain = inst.get("fxchain", {}) + if str(fxchain.get("enabled")) == "1": + num_fx = int(fxchain.get("numofeffects", 0)) + score += num_fx * W_FX + stats["num_effects"] += num_fx + + if score <= 15: + estrelas = 1 + elif score <= 40: + estrelas = 2 + elif score <= 80: + estrelas = 3 + elif score <= 120: + estrelas = 4 + else: + estrelas = 5 + + return {"score_total": round(score, 2), "estrelas": estrelas, "detalhes": stats} + + +# ================= NÚCLEO DE ANÁLISE ================= + +def detectar_estrutura(audio_vec, sample_rate, duration): + try: + if duration < 30: + return [0.0, round(duration, 2)] + + frame_size = 2048 + hop_size = 512 + + rms_algo = es.RMS() + flux_algo = es.Flux() + w = es.Windowing(type="hann") + spec = es.Spectrum() + + rms_curve = [] + flux_curve = [] + + for frame in es.FrameGenerator( + audio_vec, frameSize=frame_size, hopSize=hop_size + ): + s = spec(w(frame)) + rms_curve.append(rms_algo(frame)) + flux_curve.append(flux_algo(s)) + + min_len = min(len(rms_curve), len(flux_curve)) + rms_curve = np.array(rms_curve[:min_len]) + flux_curve = np.array(flux_curve[:min_len]) + + if np.max(rms_curve) > 0: + rms_curve = rms_curve / np.max(rms_curve) + if np.max(flux_curve) > 0: + flux_curve = flux_curve / np.max(flux_curve) + + novelty = rms_curve * 0.5 + flux_curve * 0.5 + segundos_por_frame = hop_size / sample_rate + + pontos_mudanca = [0.0] + window_frames = int(10 / segundos_por_frame) + step_frames = int(5 / segundos_por_frame) + total_frames = len(novelty) + + for i in range(0, total_frames - window_frames, step_frames): + janela = novelty[i : i + window_frames] + if np.std(janela) > 0.1: + pico_idx = i + np.argmax(janela) + tempo = round(pico_idx * segundos_por_frame, 2) + if tempo - pontos_mudanca[-1] > 15: + pontos_mudanca.append(tempo) + + pontos_mudanca.append(round(duration, 2)) + return pontos_mudanca + + except Exception as e: + logging.error(f"Erro estrutura nova: {e}") + return [0.0, round(duration, 2)] + + +def analisar_faixa(caminho_arquivo, embedding_model, classifier_model, classes_raw): + # --- AUDITORIA: INÍCIO DO TIMER --- + inicio_timer = time.perf_counter() + + tamanho_arquivo = os.path.getsize(caminho_arquivo) + pode_processar, motivo = verificar_memoria_segura(tamanho_arquivo) + + if not pode_processar: + logging.warning(f"SKIPPED {os.path.basename(caminho_arquivo)}: {motivo}") + return None, 0, 0 + + nome_arquivo = os.path.basename(caminho_arquivo) + chave_busca = normalizar_chave(nome_arquivo) + dados_humano = DATABASE_YAML.get(chave_busca) + + metadata = { + "arquivo": nome_arquivo, + "caminho": caminho_arquivo, + "dados_projeto": {}, + "analise_tecnica": {}, + "analise_ia": {}, + } + + if dados_humano: + metadata["dados_projeto"] = { + "titulo": dados_humano.get("original_title") or dados_humano.get("file"), + "bpm_original": dados_humano.get("bpm"), + "tags_yaml": dados_humano.get("tags"), + "autor": dados_humano.get("author", "Desconhecido"), + } + else: + metadata["analise_tecnica"]["complexidade"] = { + "score_total": 0, + "estrelas": 0, + "detalhes": "YAML não encontrado", + } + + duracao_real = 0.0 + + # 1. ANÁLISE TÉCNICA E EXTRAÇÃO ÚNICA (HQ 44.1kHz) + try: + # Lê do HD APENAS UMA VEZ + loader_hq = es.MonoLoader(filename=caminho_arquivo, sampleRate=44100) + audio_hq = loader_hq() + + if len(audio_hq) > (MAX_DURACAO_ANALISE * 44100): + audio_hq = audio_hq[: int(MAX_DURACAO_ANALISE * 44100)] + + duracao_real = len(audio_hq) / 44100.0 + audio_hq_vec = essentia.array(audio_hq) + + rhythm = es.RhythmExtractor2013(method="multifeature") + bpm, _, conf, _, _ = rhythm(audio_hq_vec) + + key_ex = es.KeyExtractor() + key, scale, _ = key_ex(audio_hq_vec) + + loudness = es.Loudness()(audio_hq_vec) + estrutura = detectar_estrutura(audio_hq_vec, 44100, duracao_real) + + bpm_final = round(bpm, 1) + origem = "algoritmo" + if dados_humano and dados_humano.get("bpm"): + try: + bpm_str = str(dados_humano.get("bpm")).replace("'", "").replace('"', "") + b = float(bpm_str) + if b > 0: + bpm_final = b + origem = "yaml (humano)" + except: + pass + + metadata["analise_tecnica"].update( + { + "bpm": bpm_final, + "origem_bpm": origem, + "tom": key, + "escala": scale, + "intensidade_db": round(loudness, 2), + "estrutura_segundos": estrutura, + "duracao_detectada": round(duracao_real, 2), + } + ) + + metadata["analise_tecnica"]["complexidade"] = ( + calcular_complexidade(dados_humano) if dados_humano else None + ) + + except Exception as e: + logging.error(f"Erro math {nome_arquivo}: {e}") + metadata["analise_tecnica"]["erro"] = str(e) + return metadata, 0, 0 + + # 2. ANÁLISE IA (LQ 16kHz via Resample da RAM - Resolve Leitura Dupla de HD) + if embedding_model and classifier_model and classes_raw: + try: + resample_algo = es.Resample(inputSampleRate=44100, outputSampleRate=16000) + audio_lq_vec = resample_algo(audio_hq_vec) + + embeddings = embedding_model(audio_lq_vec) + predictions = classifier_model(embeddings) + + avg_preds = np.mean(predictions, axis=0) + top_indices = np.argsort(avg_preds)[::-1][:10] + + tags_detectadas = [] + genero_pai = "Unknown" + + for i, idx in enumerate(top_indices): + score = float(avg_preds[idx]) + if score < 0.05: + continue + + nome_full = classes_raw[idx] + partes = nome_full.split("---") + tag_limpa = partes[-1] + + if i == 0: + genero_pai = partes[0] if len(partes) > 0 else tag_limpa + + tags_detectadas.append({"tag": tag_limpa, "score": round(score, 3)}) + + metadata["analise_ia"] = { + "genero_macro": genero_pai, + "estilo_principal": tags_detectadas[0]["tag"] if tags_detectadas else "Unknown", + "nuvem_tags": tags_detectadas, + } + + del audio_lq_vec + del embeddings + + except Exception as e: + msg = f"Erro IA Pipeline: {str(e)}" + logging.error(msg) + metadata["analise_ia"] = {"erro": msg} + else: + metadata["analise_ia"] = {"status": "Modelos nao carregados"} + + # Limpeza Limpa + del audio_hq + del audio_hq_vec + gc.collect() + + tempo_gasto = time.perf_counter() - inicio_timer + return metadata, duracao_real, tempo_gasto + + +# ================= MAIN (Multiprocessing) ================= +def worker_analise(caminho): + # Processa usando as instâncias dos modelos carregadas na thread mestre + global GLOBAL_EMBEDDING, GLOBAL_CLASSIFIER, GLOBAL_CLASSES + return analisar_faixa(caminho, GLOBAL_EMBEDDING, GLOBAL_CLASSIFIER, GLOBAL_CLASSES) + +def main(): + print("\n--- INICIANDO PROCESSADOR + AUDITORIA ---") + + # Referencia variávies globais do script para alimentar os childs process do pool + global DATABASE_YAML, GLOBAL_EMBEDDING, GLOBAL_CLASSIFIER, GLOBAL_CLASSES + + DATABASE_YAML = carregar_database_yaml() + + ja_processados, lista_resultados = carregar_checkpoint() + + auditor = Auditoria(ARQUIVO_AUDITORIA) + + if not os.path.exists(PASTA_WAVS): + print("CRÍTICO: Pasta WAV não encontrada!") + return + + todos = [f for f in os.listdir(PASTA_WAVS) if f.lower().endswith((".wav", ".ogg"))] + a_fazer = [os.path.join(PASTA_WAVS, f) for f in todos if f not in ja_processados] + + total = len(a_fazer) + print(f"Total Arquivos: {len(todos)} | Novos a Fazer: {total}") + print(f"Auditoria será salva em: {ARQUIVO_AUDITORIA}") + + # === CARREGAMENTO MODELOS GLOBAIS === + path_embed = MODELO_EMBEDDING if os.path.exists(MODELO_EMBEDDING) else os.path.join(BASE_DIR, MODELO_EMBEDDING) + path_class = MODELO_CLASSIFIER if os.path.exists(MODELO_CLASSIFIER) else os.path.join(BASE_DIR, MODELO_CLASSIFIER) + path_json = MODELO_CLASSES if os.path.exists(MODELO_CLASSES) else os.path.join(BASE_DIR, MODELO_CLASSES) + + if os.path.exists(path_embed) and os.path.exists(path_class) and os.path.exists(path_json): + print("--- Carregando Modelos de IA ---") + try: + with open(path_json, "r") as f: + GLOBAL_CLASSES = json.load(f)["classes"] + + print("[1/2] Carregando Extrator de Embeddings...") + if hasattr(es, "TensorflowPredictEffnetDiscogs"): + GLOBAL_EMBEDDING = es.TensorflowPredictEffnetDiscogs(graphFilename=path_embed, output="PartitionedCall:1") + else: + print(" -> Usando TensorflowPredict genérico.") + GLOBAL_EMBEDDING = es.TensorflowPredict(graphFilename=path_embed, input="serving_default_model_Placeholder", output="PartitionedCall:1") + print(" -> Embeddings carregados com sucesso.") + + GLOBAL_CLASSIFIER = es.TensorflowPredict2D(graphFilename=path_class, input="serving_default_model_Placeholder", output="PartitionedCall:0") + print("[2/2] Classificador de Gênero carregado.") + + except Exception as e: + print(f"ERRO CRÍTICO AO CARREGAR IA: {e}") + return + else: + print("Arquivos de modelo incompletos.") + return + + if total == 0: + print("Tudo atualizado.") + return + + # ProcessPoolExecutor lidará com N filas ao mesmo tempo + max_workers = max(1, multiprocessing.cpu_count() - 1) + print(f"-> Iniciando processamento paralelo com {max_workers} workers.") + contagem_sessao = 0 + + try: + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + resultados_futuros = {executor.submit(worker_analise, caminho): caminho for caminho in a_fazer} + + # O as_completed processa os que terminam mais rápido em vez de travar na ordem + for future in tqdm(concurrent.futures.as_completed(resultados_futuros), total=len(a_fazer), unit="track"): + try: + res, duracao, tempo_gasto = future.result() + caminho = resultados_futuros[future] + + if res: + salvar_progresso(res) + lista_resultados.append(res) + + tamanho = os.path.getsize(caminho) + auditor.registrar_processamento(res['arquivo'], tamanho, duracao, tempo_gasto) + + contagem_sessao += 1 + auditor.verificar_marco(contagem_sessao) + + except Exception as e: + logging.error(f"Erro ao processar um dos arquivos no worker: {e}") + + except KeyboardInterrupt: + print("\nParando paralelismo graciosamente...") + + finally: + auditor.fechar() # Fecha o CSV com segurança + + tempo_total_sessao = time.time() - auditor.inicio_global + print("\n--- FIM DO PROCESSAMENTO ---") + print(f"Tempo Total: {str(timedelta(seconds=int(tempo_total_sessao)))}") + print("Gerando JSON final consolidado...") + + with open(ARQUIVO_FINAL_JSON, "w", encoding="utf-8") as f: + json.dump(lista_resultados, f, indent=4, ensure_ascii=False) + + print(f"Dados JSON: {ARQUIVO_FINAL_JSON}") + print(f"Auditoria CSV: {ARQUIVO_AUDITORIA}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/classificacao/classificacao_mestre.py b/scripts/classificacao/classificacao_mestre.py new file mode 100644 index 0000000..e4f1132 --- /dev/null +++ b/scripts/classificacao/classificacao_mestre.py @@ -0,0 +1,457 @@ +import yaml +import json +import os +import logging +import csv +import unicodedata +import time +import psutil +import glob # Import necessário para ler múltiplos arquivos +from datetime import datetime, timedelta + +# --- CONFIGURAÇÕES GERAIS --- +BASE_DIR = "/var/www/html/trens/src_mmpSearch" +LOG_DIR = os.path.join(BASE_DIR, "logs/classificacao_pedagogica") +os.makedirs(LOG_DIR, exist_ok=True) +TIMESTAMP = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + +# Configuração de Log +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - CLASSIFICADOR - %(levelname)s - %(message)s", + handlers=[ + logging.FileHandler(os.path.join(LOG_DIR, f"log_pedagogico_{TIMESTAMP}.log")), + logging.StreamHandler(), + ], +) + + +# --- ARQUIVOS DE ENTRADA/SAÍDA --- +def encontrar_caminho(nome_arquivo): + """Procura o arquivo em locais comuns do projeto.""" + candidatos = [ + os.path.join(BASE_DIR, "saida_analises", nome_arquivo), + os.path.join(BASE_DIR, "data", nome_arquivo), + os.path.join(BASE_DIR, nome_arquivo), + nome_arquivo, + ] + for c in candidatos: + if os.path.exists(c): + return c + return nome_arquivo + + +# --- MUDANÇA AQUI: Aponta para a pasta dos lotes --- +PASTA_LOTES_YAML = os.path.join(BASE_DIR, "lotes_yaml") +# ARQUIVO_YAML = "all.yml" <-- Removido/Comentado + +ARQUIVO_AUDIO_INPUT = encontrar_caminho("db_final_completo.json") +ARQUIVO_FINAL_OUTPUT = os.path.join( + BASE_DIR, "saida_analises", "db_projetos_classificados.json" +) +ARQUIVO_AUDITORIA = os.path.join( + BASE_DIR, "saida_analises", "audit_classificacao_pedagogica.csv" +) + +# --- LISTAS DE REFERÊNCIA --- +PLUGINS_NATIVOS = [ + "tripleoscillator", + "zynaddsubfx", + "sfxr", + "organic", + "audiofileprocessor", + "lb302", + "kicker", + "watsyn", + "bitinvader", + "freeboy", + "mallets", + "vibed", +] + + +# --- CLASSE DE AUDITORIA DE PERFORMANCE --- +class AuditoriaPerformance: + def __init__(self): + self.inicio_global = time.time() + self.process = psutil.Process(os.getpid()) + self.marcos = [1, 10, 100, 500, 1000, 2000, 5000] + + def get_metricas(self): + """Retorna uso atual de RAM (MB) e CPU (%)""" + try: + ram_mb = self.process.memory_info().rss / (1024 * 1024) + cpu_pct = self.process.cpu_percent(interval=None) + return round(ram_mb, 2), cpu_pct + except: + return 0.0, 0.0 + + def verificar_marco(self, contagem): + if contagem in self.marcos: + tempo_decorrido = time.time() - self.inicio_global + logging.info( + f"--- MARCO: {contagem} projetos processados em {str(timedelta(seconds=int(tempo_decorrido)))} ---" + ) + + +# --- FUNÇÕES AUXILIARES --- + + +def normalizar_chave(texto): + if not texto: + return "" + s = str(texto) + if "." in s: + s = os.path.splitext(s)[0] + s = unicodedata.normalize("NFKD", s).encode("ASCII", "ignore").decode("ASCII") + return s.lower().replace(" ", "").replace("-", "").replace("_", "").strip() + + +def calcular_nivel_ponderado( + num_tracks, tem_automacao, usa_zyn, qtd_secoes_audio, fx_criativos +): + pontos = 0 + + # 1. Volume + if num_tracks >= 8: + pontos += 1 + if num_tracks >= 16: + pontos += 2 + if num_tracks >= 32: + pontos += 3 + if num_tracks >= 64: + pontos += 4 + elif num_tracks >= 128: + pontos += 5 + + # 2. Técnica + if tem_automacao: + pontos += 2 + if usa_zyn: + pontos += 1.5 + if fx_criativos: + pontos += 1 + + # 3. Estrutura (Áudio) + if qtd_secoes_audio >= 3: + pontos += 1.5 + elif qtd_secoes_audio >= 5: + pontos += 3 + elif qtd_secoes_audio >= 10: + pontos += 5 + + if pontos < 3: + return "Iniciante" + if pontos < 6: + return "Intermediário" + return "Avançado" + + +def analisar_tags_pedagogicas(tracks, bpm, qtd_secoes): + tags = [] + tem_kicker = False + tem_zyn = False + tem_automacao = False + tem_fx_chain = False + rolagem_trap = False + + for t in tracks: + inst = t.get("instrumenttrack") + if not inst: + continue + + nome = inst.get("name", "").lower() + plugin = inst.get("plugin_name", "").lower() + + if plugin == "kicker": + tem_kicker = True + if plugin == "zynaddsubfx": + tem_zyn = True + if "automation" in str(inst).lower(): + tem_automacao = True + + fx = inst.get("fxchain", {}) + if fx and int(fx.get("numofeffects", 0)) > 0: + tem_fx_chain = True + + patterns = inst.get("patterns", []) + if patterns: + for pat in patterns: + steps = pat.get("steps", []) + consecutivos = 0 + for step in steps: + val = 1 if step else 0 + consecutivos = (consecutivos + 1) if val else 0 + if consecutivos >= 3 and ("hat" in nome): + rolagem_trap = True + + if tem_kicker: + tags.append("Síntese de Bateria") + if tem_zyn: + tags.append("Design de Som Avançado") + if tem_automacao: + tags.append("Automação") + if tem_fx_chain: + tags.append("Mixagem com FX") + + if rolagem_trap and bpm > 115: + tags.append("Trap Rhythm") + if bpm < 100 and not rolagem_trap: + tags.append("Boombap Groove") + + if qtd_secoes >= 4: + tags.append("Arranjo Completo") + elif qtd_secoes <= 1: + tags.append("Loop Estático") + else: + tags.append("Estrutura Simples") + + return tags, tem_automacao, tem_zyn, tem_fx_chain + + +# --- FUNÇÃO PARA CARREGAR LOTES YAML --- +def carregar_todos_yamls(pasta_lotes): + """Lê todos os arquivos YAML da pasta especificada e retorna uma lista única.""" + logging.info(f"Buscando arquivos YAML em: {pasta_lotes}") + + padrao = os.path.join(pasta_lotes, "*.yml") + arquivos = sorted(glob.glob(padrao)) + + todos_projetos = [] + + if not arquivos: + logging.warning("Nenhum arquivo YAML encontrado na pasta de lotes.") + # Tenta fallback para o arquivo único antigo se existir + caminho_unico = encontrar_caminho("all.yml") + if os.path.exists(caminho_unico): + logging.info(f"Fallback: Carregando arquivo único {caminho_unico}") + with open(caminho_unico, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + return [] + + for arq in arquivos: + try: + logging.info(f"Carregando lote: {os.path.basename(arq)}") + with open(arq, "r", encoding="utf-8") as f: + lote = yaml.safe_load(f) + if lote: + # Garante que seja uma lista + if isinstance(lote, list): + todos_projetos.extend(lote) + elif isinstance(lote, dict): + todos_projetos.append(lote) + except Exception as e: + logging.error(f"Erro ao ler arquivo {arq}: {e}") + + logging.info(f"Total de projetos carregados dos lotes: {len(todos_projetos)}") + return todos_projetos + + +# --- MAIN --- + + +def main(): + logging.info("--- INICIANDO CLASSIFICADOR PEDAGÓGICO (COM AUDITORIA E LOTES) ---") + + auditor = AuditoriaPerformance() + + # 1. Carregar YAML (Adaptado para Lotes) + if not os.path.exists(PASTA_LOTES_YAML): + logging.warning(f"Pasta de lotes não encontrada: {PASTA_LOTES_YAML}") + # Tenta criar se não existir (embora deva existir com arquivos) + # os.makedirs(PASTA_LOTES_YAML, exist_ok=True) + + dados_projetos = carregar_todos_yamls(PASTA_LOTES_YAML) + + if not dados_projetos: + logging.critical("Nenhum dado de projeto carregado. Encerrando.") + return + + # 2. Carregar JSON de Áudio + lookup_audio = {} + total_audio_carregado = 0 + + if os.path.exists(ARQUIVO_AUDIO_INPUT): + logging.info(f"Lendo Análise de Áudio: {ARQUIVO_AUDIO_INPUT}") + with open(ARQUIVO_AUDIO_INPUT, "r", encoding="utf-8") as f: + lista_audio = json.load(f) + for item in lista_audio: + chave = normalizar_chave(item.get("arquivo", "")) + if chave: + lookup_audio[chave] = item + total_audio_carregado = len(lista_audio) + else: + logging.warning(f"Arquivo de áudio {ARQUIVO_AUDIO_INPUT} não encontrado.") + + db_final = [] + estatisticas = {"Iniciante": 0, "Intermediário": 0, "Avançado": 0, "Total": 0} + + # --- PREPARAR CSV DE AUDITORIA --- + with open(ARQUIVO_AUDITORIA, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.writer(csvfile) + # Cabeçalho expandido com métricas de performance + writer.writerow( + [ + "Timestamp", + "ID_Projeto", + "Audio_Encontrado", + "Qtd_Secoes", + "Nivel", + "Genero_Final", + "Tempo_Proc_ms", + "RAM_Uso_MB", + "Tags", + ] + ) + + logging.info(f"Processando {len(dados_projetos)} projetos...") + + contagem_processados = 0 + + for proj in dados_projetos: + if not proj or not isinstance(proj, dict): + continue + + # --- INÍCIO TIMER INDIVIDUAL --- + inicio_item = time.perf_counter() + + nome_arquivo = proj.get("file", "") + if not nome_arquivo: + continue + + try: + # Lógica de Classificação + chave_proj = normalizar_chave(nome_arquivo) + dados_audio = lookup_audio.get(chave_proj, {}) + + analise_tec = dados_audio.get("analise_tecnica", {}) + analise_ia = dados_audio.get("analise_ia", {}) + + raw_bpm = proj.get("bpm", 120) + try: + bpm_projeto = float(raw_bpm) + except: + bpm_projeto = 120.0 + + est_seg = analise_tec.get("estrutura_segundos", []) + qtd_secoes = len(est_seg) if isinstance(est_seg, list) else 1 + + tracks = proj.get("tracks", []) + tags, tem_auto, tem_zyn, tem_fx = analisar_tags_pedagogicas( + tracks, bpm_projeto, qtd_secoes + ) + + plugins_usados = set() + usa_vst_externo = False + for t in tracks: + inst_track = t.get("instrumenttrack", {}) + if not inst_track: + continue + p = inst_track.get("plugin_name", "") + if p: + p_norm = p.lower() + plugins_usados.add(p_norm) + if p_norm not in PLUGINS_NATIVOS: + usa_vst_externo = True + + nivel = calcular_nivel_ponderado( + len(tracks), tem_auto, tem_zyn, qtd_secoes, tem_fx + ) + estatisticas[nivel] += 1 + estatisticas["Total"] += 1 + + genero = "Desconhecido" + if analise_ia and "estilo_principal" in analise_ia: + genero = analise_ia["estilo_principal"] + elif ( + "genero_macro" in analise_ia + and analise_ia["genero_macro"] != "Unknown" + ): + genero = analise_ia["genero_macro"] + + if genero == "Desconhecido" or genero == "Unknown": + if "Trap Rhythm" in tags: + genero = "Trap (Rule-based)" + elif bpm_projeto > 120: + genero = "Electronic (Rule-based)" + else: + genero = "HipHop/Downtempo (Rule-based)" + + # Objeto Final + item = { + "id": nome_arquivo, + "titulo": proj.get("original_title", nome_arquivo), + "match_audio": bool(dados_audio), + "pedagogia": { + "nivel": nivel, + "tags_aprendizado": tags, + "plugins_principais": list(plugins_usados), + "requer_vst_externo": usa_vst_externo, + }, + "tecnica": { + "bpm": bpm_projeto, + "tom": analise_tec.get("tom", "N/A"), + "escala": analise_tec.get("escala", "N/A"), + "estrutura_qtd_secoes": qtd_secoes, + "intensidade_media": analise_tec.get("intensidade_db", "N/A"), + }, + "classificacao": { + "genero": genero, + "mood": "Energético" + if analise_tec.get("intensidade_db", -20) > -10 + else "Suave", + }, + } + + db_final.append(item) + + # --- FIM TIMER INDIVIDUAL --- + tempo_ms = (time.perf_counter() - inicio_item) * 1000 + ram_atual, _ = auditor.get_metricas() + ts_agora = datetime.now().strftime("%H:%M:%S") + + # Registra Auditoria com Performance + writer.writerow( + [ + ts_agora, + nome_arquivo, + "SIM" if dados_audio else "NAO", + qtd_secoes, + nivel, + genero, + f"{tempo_ms:.3f}", # Tempo em milissegundos com 3 casas + f"{ram_atual:.2f}", # RAM em MB + "|".join(tags), + ] + ) + + contagem_processados += 1 + auditor.verificar_marco(contagem_processados) + + except Exception as e: + logging.error( + f"Erro ao processar projeto '{proj.get('file', 'unknown')}': {e}" + ) + continue + + # Salvar Resultado JSON + logging.info(f"Salvando JSON Final: {ARQUIVO_FINAL_OUTPUT}") + with open(ARQUIVO_FINAL_OUTPUT, "w", encoding="utf-8") as f: + json.dump(db_final, f, indent=4, ensure_ascii=False) + + tempo_total = time.time() - auditor.inicio_global + + logging.info("--- RELATÓRIO FINAL ---") + logging.info(f"Tempo Total de Execução: {str(timedelta(seconds=int(tempo_total)))}") + logging.info(f"Total Processado: {estatisticas['Total']}") + logging.info(f"Iniciantes: {estatisticas['Iniciante']}") + logging.info(f"Intermediários: {estatisticas['Intermediário']}") + logging.info(f"Avançados: {estatisticas['Avançado']}") + logging.info( + f"Taxa de Casamento com Áudio: {len(lookup_audio)}/{total_audio_carregado}" + ) + logging.info(f"Auditoria salva em: {ARQUIVO_AUDITORIA}") + + +if __name__ == "__main__": + main() diff --git a/scripts/classificacao/classificador_mestre.py b/scripts/classificacao/classificador_mestre.py new file mode 100644 index 0000000..b19bdd1 --- /dev/null +++ b/scripts/classificacao/classificador_mestre.py @@ -0,0 +1,274 @@ +import yaml +import json +import os +import logging + +# --- CONFIGURAÇÕES DE LOG --- +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - CLASSIFICADOR - %(levelname)s - %(message)s", + handlers=[ + logging.FileHandler("classificacao_pedagogica.log"), + logging.StreamHandler(), + ], +) + +# --- CONFIGURAÇÕES --- +ARQUIVO_YAML = "all.yml" +ARQUIVO_AUDIO = "analise_audio.json" +ARQUIVO_FINAL = "db_projetos_classificados.json" + +PLUGINS_NATIVOS = [ + "tripleoscillator", + "zynaddsubfx", + "sfxr", + "organic", + "audiofileprocessor", + "lb302", + "kicker", + "watsyn", + "bitinvader", + "freeboy", + "mallets", + "vibed", +] + +# --- LÓGICA PEDAGÓGICA AVANÇADA --- + + +def calcular_nivel_ponderado( + num_tracks, tem_automacao, plugins_complexos, qtd_secoes_audio, fx_criativos +): + """ + Calcula nível pedagógico (Iniciante, Intermediário, Avançado) + baseado em esforço de engenharia e complexidade musical. + """ + pontos = 0 + + # 1. Volume de Trabalho + if num_tracks >= 8: + pontos += 1 + if num_tracks >= 16: + pontos += 1 + + # 2. Profundidade Técnica + if tem_automacao: + pontos += 2 # Automação é sinal forte de polimento + if plugins_complexos: + pontos += 1.5 # Usar ZynAddSubFx requer estudo + if fx_criativos: + pontos += 1 # Usar efeitos além do básico + + # 3. Estrutura Musical (Vinda do Essentia) + # Se o áudio tem > 3 seções detectadas, há arranjo (não é só loop) + if qtd_secoes_audio >= 3: + pontos += 2 + elif qtd_secoes_audio >= 5: + pontos += 3 + + # Classificação + if pontos < 3: + return "Iniciante" + if pontos < 6: + return "Intermediário" + return "Avançado" + + +def analisar_tags_pedagogicas(tracks, bpm, qtd_secoes, tom, escala): + tags = [] + + # Flags de estado técnico + tem_zyn = False + tem_automacao = False + tem_fx_chain = False + qtd_nativos = 0 + qtd_vst_externos = 0 + tem_audio_processor = False + + for t in tracks: + inst = t.get("instrumenttrack") + if not inst: + continue + + plugin = inst.get("plugin_name", "").lower() + + # Mapeamento de Plugins e Dependências + if plugin in PLUGINS_NATIVOS: + qtd_nativos += 1 + elif plugin == "vestige": + qtd_vst_externos += 1 + + if plugin == "zynaddsubfx": + tem_zyn = True + + if plugin == "audiofileprocessor": + tem_audio_processor = True + + if "automation" in str(inst).lower(): + tem_automacao = True + + fx = inst.get("fxchain", {}) + if fx and int(fx.get("numofeffects", 0)) > 0: + tem_fx_chain = True + + # --- 1. Estética da Escassez e Modulação Marginal --- + if qtd_nativos > 0 and qtd_vst_externos == 0: + tags.append("Estética da Escassez: Síntese Nativa") + + if tem_automacao and qtd_vst_externos == 0: + tags.append("Design de Som Marginal (Automação como Timbre)") + + # --- 2. Taxonomia de Bloom (Carga Cognitiva) --- + if tem_zyn or tem_fx_chain: + tags.append("Bloom: Criação (Sound Design Complexo)") + elif tem_audio_processor and not tem_automacao: + tags.append("Bloom: Aplicação (Sample Inalterado)") + + # --- 3. Fluência Estrutural (Macroforma Musical) --- + if qtd_secoes >= 3: + tags.append("Fluência Estrutural: Arranjo Completo (Macroforma)") + else: + tags.append("Fluência Estrutural: Loop Estático") + + # --- 4. Acessibilidade Ergonômica (Música de Computador) --- + if tom == "A" and escala == "minor": + tags.append("Ergonomia Autodidata (Teclado QWERTY / Lá Menor)") + + # --- 5. Herança Cultural / Andamento --- + if 80 <= bpm <= 95: + tags.append("Cadência Clássica (Boom Bap)") + elif bpm >= 130: + tags.append("Estética Trap/Funk/Eletrônica / Alta Energia") + + return tags + + +def main(): + logging.info("Iniciando Classificador Mestre...") + + # 1. Carregar YAML + if not os.path.exists(ARQUIVO_YAML): + logging.critical(f"{ARQUIVO_YAML} não encontrado.") + return + + with open(ARQUIVO_YAML, "r", encoding="utf-8") as f: + dados_projetos = yaml.safe_load(f) + + # 2. Carregar JSON de Áudio + lookup_audio = {} + if os.path.exists(ARQUIVO_AUDIO): + with open(ARQUIVO_AUDIO, "r", encoding="utf-8") as f: + lista_audio = json.load(f) + for item in lista_audio: + # Normaliza chave de busca (nome do arquivo sem extensão) + nome_base = os.path.splitext(item["arquivo"])[0] + lookup_audio[nome_base] = item + else: + logging.warning( + "JSON de áudio não encontrado. Classificação será apenas estática." + ) + + db_final = [] + + logging.info(f"Processando {len(dados_projetos)} entradas do YAML...") + + for proj in dados_projetos: + if not proj: + continue + + try: + nome_arquivo = proj.get("file", "") + if not nome_arquivo: + continue + + # Tenta casar com dados de áudio + # Nota: Seu YAML usa nomes como 'trap-fyrebreak', o arquivo de audio deve ser 'trap-fyrebreak.wav' + # O lookup_audio já está sem extensão. + dados_audio = lookup_audio.get(nome_arquivo, {}) + analise_tec = dados_audio.get("analise_tecnica", {}) + analise_ia = dados_audio.get("analise_ia", {}) + + # Dados Técnicos + raw_bpm = proj.get("bpm", 120) + bpm_projeto = ( + float(raw_bpm) if str(raw_bpm).replace(".", "", 1).isdigit() else 120.0 + ) + + # Feature nova: Quantidade de seções (Intro, Verso...) + qtd_secoes = analise_tec.get("qtd_secoes_detectadas", 1) + + # Análise Estática (Tracks e Plugins) + tracks = proj.get("tracks", []) + tags, tem_auto, tem_zyn, tem_fx = analisar_tags_pedagogicas( + tracks=proj.get("tracks", []), + bpm=bpm_projeto, + qtd_secoes=qtd_secoes, + tom=analise_tec.get("tom", "N/A"), + escala=analise_tec.get("escala", "N/A"), + ) + + # Verifica plugins externos + plugins_usados = set() + usa_vst_externo = False + for t in tracks: + p = t.get("instrumenttrack", {}).get("plugin_name", "") + if p: + plugins_usados.add(p) + if p.lower() not in PLUGINS_NATIVOS: + usa_vst_externo = True + + # Cálculo de Nível (Nova Lógica) + nivel = calcular_nivel_ponderado( + len(tracks), tem_auto, tem_zyn, qtd_secoes, tem_fx + ) + + # Define Gênero (IA tem prioridade, depois regras manuais) + genero = analise_ia.get("genero_predito", "Desconhecido") + if genero == "Desconhecido" or analise_ia.get("confianca_genero", 0) < 0.4: + # Fallback manual simples + if "Trap Rhythm" in tags: + genero = "Trap" + elif bpm_projeto > 120: + genero = "Electronic" + else: + genero = "HipHop/Downtempo" + + # Montagem do Objeto + item = { + "id": nome_arquivo, + "titulo": proj.get("original_title", nome_arquivo), + "pedagogia": { + "nivel": nivel, + "tags_aprendizado": tags, + "plugins_principais": list(plugins_usados), + "requer_vst_externo": usa_vst_externo, + }, + "tecnica": { + "bpm": bpm_projeto, + "tom": analise_tec.get("tom", "N/A"), + "escala": analise_tec.get("escala", "N/A"), + "estrutura_detectada": f"{qtd_secoes} seções distintas", + }, + "classificacao": { + "genero": genero, + "mood": "Energético" + if analise_tec.get("intensidade", -20) > -10 + else "Suave", + }, + } + + db_final.append(item) + + except Exception as e: + logging.error(f"Erro ao processar projeto '{proj.get('file')}': {e}") + continue + + # Salvar Resultado + with open(ARQUIVO_FINAL, "w", encoding="utf-8") as f: + json.dump(db_final, f, indent=4, ensure_ascii=False) + + logging.info(f"Sucesso! {len(db_final)} projetos classificados e salvos.") + + +if __name__ == "__main__": + main() diff --git a/scripts/classificacao/discogs-effnet-bs64-1.json b/scripts/classificacao/discogs-effnet-bs64-1.json new file mode 100644 index 0000000..5cbe77b --- /dev/null +++ b/scripts/classificacao/discogs-effnet-bs64-1.json @@ -0,0 +1,468 @@ +{ + "name": "EffnetDiscogs", + "type": "Music style classification and embeddings", + "link": "https://essentia.upf.edu/models/music-style-classification/discogs-effnet/discogs-effnet-bs64-1.pb", + "version": "1", + "description": "Prediction of the top-400 music styles in the Discogs-4M dataset (unreleased)", + "author": "Pablo Alonso", + "email": "pablo.alonso@upf.edu", + "release_date": "2022-02-17", + "framework": "tensorflow", + "framework_version": "2.8.0", + "classes": [ + "Blues---Boogie Woogie", + "Blues---Chicago Blues", + "Blues---Country Blues", + "Blues---Delta Blues", + "Blues---Electric Blues", + "Blues---Harmonica Blues", + "Blues---Jump Blues", + "Blues---Louisiana Blues", + "Blues---Modern Electric Blues", + "Blues---Piano Blues", + "Blues---Rhythm & Blues", + "Blues---Texas Blues", + "Brass & Military---Brass Band", + "Brass & Military---Marches", + "Brass & Military---Military", + "Children's---Educational", + "Children's---Nursery Rhymes", + "Children's---Story", + "Classical---Baroque", + "Classical---Choral", + "Classical---Classical", + "Classical---Contemporary", + "Classical---Impressionist", + "Classical---Medieval", + "Classical---Modern", + "Classical---Neo-Classical", + "Classical---Neo-Romantic", + "Classical---Opera", + "Classical---Post-Modern", + "Classical---Renaissance", + "Classical---Romantic", + "Electronic---Abstract", + "Electronic---Acid", + "Electronic---Acid House", + "Electronic---Acid Jazz", + "Electronic---Ambient", + "Electronic---Bassline", + "Electronic---Beatdown", + "Electronic---Berlin-School", + "Electronic---Big Beat", + "Electronic---Bleep", + "Electronic---Breakbeat", + "Electronic---Breakcore", + "Electronic---Breaks", + "Electronic---Broken Beat", + "Electronic---Chillwave", + "Electronic---Chiptune", + "Electronic---Dance-pop", + "Electronic---Dark Ambient", + "Electronic---Darkwave", + "Electronic---Deep House", + "Electronic---Deep Techno", + "Electronic---Disco", + "Electronic---Disco Polo", + "Electronic---Donk", + "Electronic---Downtempo", + "Electronic---Drone", + "Electronic---Drum n Bass", + "Electronic---Dub", + "Electronic---Dub Techno", + "Electronic---Dubstep", + "Electronic---Dungeon Synth", + "Electronic---EBM", + "Electronic---Electro", + "Electronic---Electro House", + "Electronic---Electroclash", + "Electronic---Euro House", + "Electronic---Euro-Disco", + "Electronic---Eurobeat", + "Electronic---Eurodance", + "Electronic---Experimental", + "Electronic---Freestyle", + "Electronic---Future Jazz", + "Electronic---Gabber", + "Electronic---Garage House", + "Electronic---Ghetto", + "Electronic---Ghetto House", + "Electronic---Glitch", + "Electronic---Goa Trance", + "Electronic---Grime", + "Electronic---Halftime", + "Electronic---Hands Up", + "Electronic---Happy Hardcore", + "Electronic---Hard House", + "Electronic---Hard Techno", + "Electronic---Hard Trance", + "Electronic---Hardcore", + "Electronic---Hardstyle", + "Electronic---Hi NRG", + "Electronic---Hip Hop", + "Electronic---Hip-House", + "Electronic---House", + "Electronic---IDM", + "Electronic---Illbient", + "Electronic---Industrial", + "Electronic---Italo House", + "Electronic---Italo-Disco", + "Electronic---Italodance", + "Electronic---Jazzdance", + "Electronic---Juke", + "Electronic---Jumpstyle", + "Electronic---Jungle", + "Electronic---Latin", + "Electronic---Leftfield", + "Electronic---Makina", + "Electronic---Minimal", + "Electronic---Minimal Techno", + "Electronic---Modern Classical", + "Electronic---Musique Concr\u00e8te", + "Electronic---Neofolk", + "Electronic---New Age", + "Electronic---New Beat", + "Electronic---New Wave", + "Electronic---Noise", + "Electronic---Nu-Disco", + "Electronic---Power Electronics", + "Electronic---Progressive Breaks", + "Electronic---Progressive House", + "Electronic---Progressive Trance", + "Electronic---Psy-Trance", + "Electronic---Rhythmic Noise", + "Electronic---Schranz", + "Electronic---Sound Collage", + "Electronic---Speed Garage", + "Electronic---Speedcore", + "Electronic---Synth-pop", + "Electronic---Synthwave", + "Electronic---Tech House", + "Electronic---Tech Trance", + "Electronic---Techno", + "Electronic---Trance", + "Electronic---Tribal", + "Electronic---Tribal House", + "Electronic---Trip Hop", + "Electronic---Tropical House", + "Electronic---UK Garage", + "Electronic---Vaporwave", + "Folk, World, & Country---African", + "Folk, World, & Country---Bluegrass", + "Folk, World, & Country---Cajun", + "Folk, World, & Country---Canzone Napoletana", + "Folk, World, & Country---Catalan Music", + "Folk, World, & Country---Celtic", + "Folk, World, & Country---Country", + "Folk, World, & Country---Fado", + "Folk, World, & Country---Flamenco", + "Folk, World, & Country---Folk", + "Folk, World, & Country---Gospel", + "Folk, World, & Country---Highlife", + "Folk, World, & Country---Hillbilly", + "Folk, World, & Country---Hindustani", + "Folk, World, & Country---Honky Tonk", + "Folk, World, & Country---Indian Classical", + "Folk, World, & Country---La\u00efk\u00f3", + "Folk, World, & Country---Nordic", + "Folk, World, & Country---Pacific", + "Folk, World, & Country---Polka", + "Folk, World, & Country---Ra\u00ef", + "Folk, World, & Country---Romani", + "Folk, World, & Country---Soukous", + "Folk, World, & Country---S\u00e9ga", + "Folk, World, & Country---Volksmusik", + "Folk, World, & Country---Zouk", + "Folk, World, & Country---\u00c9ntekhno", + "Funk / Soul---Afrobeat", + "Funk / Soul---Boogie", + "Funk / Soul---Contemporary R&B", + "Funk / Soul---Disco", + "Funk / Soul---Free Funk", + "Funk / Soul---Funk", + "Funk / Soul---Gospel", + "Funk / Soul---Neo Soul", + "Funk / Soul---New Jack Swing", + "Funk / Soul---P.Funk", + "Funk / Soul---Psychedelic", + "Funk / Soul---Rhythm & Blues", + "Funk / Soul---Soul", + "Funk / Soul---Swingbeat", + "Funk / Soul---UK Street Soul", + "Hip Hop---Bass Music", + "Hip Hop---Boom Bap", + "Hip Hop---Bounce", + "Hip Hop---Britcore", + "Hip Hop---Cloud Rap", + "Hip Hop---Conscious", + "Hip Hop---Crunk", + "Hip Hop---Cut-up/DJ", + "Hip Hop---DJ Battle Tool", + "Hip Hop---Electro", + "Hip Hop---G-Funk", + "Hip Hop---Gangsta", + "Hip Hop---Grime", + "Hip Hop---Hardcore Hip-Hop", + "Hip Hop---Horrorcore", + "Hip Hop---Instrumental", + "Hip Hop---Jazzy Hip-Hop", + "Hip Hop---Miami Bass", + "Hip Hop---Pop Rap", + "Hip Hop---Ragga HipHop", + "Hip Hop---RnB/Swing", + "Hip Hop---Screw", + "Hip Hop---Thug Rap", + "Hip Hop---Trap", + "Hip Hop---Trip Hop", + "Hip Hop---Turntablism", + "Jazz---Afro-Cuban Jazz", + "Jazz---Afrobeat", + "Jazz---Avant-garde Jazz", + "Jazz---Big Band", + "Jazz---Bop", + "Jazz---Bossa Nova", + "Jazz---Contemporary Jazz", + "Jazz---Cool Jazz", + "Jazz---Dixieland", + "Jazz---Easy Listening", + "Jazz---Free Improvisation", + "Jazz---Free Jazz", + "Jazz---Fusion", + "Jazz---Gypsy Jazz", + "Jazz---Hard Bop", + "Jazz---Jazz-Funk", + "Jazz---Jazz-Rock", + "Jazz---Latin Jazz", + "Jazz---Modal", + "Jazz---Post Bop", + "Jazz---Ragtime", + "Jazz---Smooth Jazz", + "Jazz---Soul-Jazz", + "Jazz---Space-Age", + "Jazz---Swing", + "Latin---Afro-Cuban", + "Latin---Bai\u00e3o", + "Latin---Batucada", + "Latin---Beguine", + "Latin---Bolero", + "Latin---Boogaloo", + "Latin---Bossanova", + "Latin---Cha-Cha", + "Latin---Charanga", + "Latin---Compas", + "Latin---Cubano", + "Latin---Cumbia", + "Latin---Descarga", + "Latin---Forr\u00f3", + "Latin---Guaguanc\u00f3", + "Latin---Guajira", + "Latin---Guaracha", + "Latin---MPB", + "Latin---Mambo", + "Latin---Mariachi", + "Latin---Merengue", + "Latin---Norte\u00f1o", + "Latin---Nueva Cancion", + "Latin---Pachanga", + "Latin---Porro", + "Latin---Ranchera", + "Latin---Reggaeton", + "Latin---Rumba", + "Latin---Salsa", + "Latin---Samba", + "Latin---Son", + "Latin---Son Montuno", + "Latin---Tango", + "Latin---Tejano", + "Latin---Vallenato", + "Non-Music---Audiobook", + "Non-Music---Comedy", + "Non-Music---Dialogue", + "Non-Music---Education", + "Non-Music---Field Recording", + "Non-Music---Interview", + "Non-Music---Monolog", + "Non-Music---Poetry", + "Non-Music---Political", + "Non-Music---Promotional", + "Non-Music---Radioplay", + "Non-Music---Religious", + "Non-Music---Spoken Word", + "Pop---Ballad", + "Pop---Bollywood", + "Pop---Bubblegum", + "Pop---Chanson", + "Pop---City Pop", + "Pop---Europop", + "Pop---Indie Pop", + "Pop---J-pop", + "Pop---K-pop", + "Pop---Kay\u014dkyoku", + "Pop---Light Music", + "Pop---Music Hall", + "Pop---Novelty", + "Pop---Parody", + "Pop---Schlager", + "Pop---Vocal", + "Reggae---Calypso", + "Reggae---Dancehall", + "Reggae---Dub", + "Reggae---Lovers Rock", + "Reggae---Ragga", + "Reggae---Reggae", + "Reggae---Reggae-Pop", + "Reggae---Rocksteady", + "Reggae---Roots Reggae", + "Reggae---Ska", + "Reggae---Soca", + "Rock---AOR", + "Rock---Acid Rock", + "Rock---Acoustic", + "Rock---Alternative Rock", + "Rock---Arena Rock", + "Rock---Art Rock", + "Rock---Atmospheric Black Metal", + "Rock---Avantgarde", + "Rock---Beat", + "Rock---Black Metal", + "Rock---Blues Rock", + "Rock---Brit Pop", + "Rock---Classic Rock", + "Rock---Coldwave", + "Rock---Country Rock", + "Rock---Crust", + "Rock---Death Metal", + "Rock---Deathcore", + "Rock---Deathrock", + "Rock---Depressive Black Metal", + "Rock---Doo Wop", + "Rock---Doom Metal", + "Rock---Dream Pop", + "Rock---Emo", + "Rock---Ethereal", + "Rock---Experimental", + "Rock---Folk Metal", + "Rock---Folk Rock", + "Rock---Funeral Doom Metal", + "Rock---Funk Metal", + "Rock---Garage Rock", + "Rock---Glam", + "Rock---Goregrind", + "Rock---Goth Rock", + "Rock---Gothic Metal", + "Rock---Grindcore", + "Rock---Grunge", + "Rock---Hard Rock", + "Rock---Hardcore", + "Rock---Heavy Metal", + "Rock---Indie Rock", + "Rock---Industrial", + "Rock---Krautrock", + "Rock---Lo-Fi", + "Rock---Lounge", + "Rock---Math Rock", + "Rock---Melodic Death Metal", + "Rock---Melodic Hardcore", + "Rock---Metalcore", + "Rock---Mod", + "Rock---Neofolk", + "Rock---New Wave", + "Rock---No Wave", + "Rock---Noise", + "Rock---Noisecore", + "Rock---Nu Metal", + "Rock---Oi", + "Rock---Parody", + "Rock---Pop Punk", + "Rock---Pop Rock", + "Rock---Pornogrind", + "Rock---Post Rock", + "Rock---Post-Hardcore", + "Rock---Post-Metal", + "Rock---Post-Punk", + "Rock---Power Metal", + "Rock---Power Pop", + "Rock---Power Violence", + "Rock---Prog Rock", + "Rock---Progressive Metal", + "Rock---Psychedelic Rock", + "Rock---Psychobilly", + "Rock---Pub Rock", + "Rock---Punk", + "Rock---Rock & Roll", + "Rock---Rockabilly", + "Rock---Shoegaze", + "Rock---Ska", + "Rock---Sludge Metal", + "Rock---Soft Rock", + "Rock---Southern Rock", + "Rock---Space Rock", + "Rock---Speed Metal", + "Rock---Stoner Rock", + "Rock---Surf", + "Rock---Symphonic Rock", + "Rock---Technical Death Metal", + "Rock---Thrash", + "Rock---Twist", + "Rock---Viking Metal", + "Rock---Y\u00e9-Y\u00e9", + "Stage & Screen---Musical", + "Stage & Screen---Score", + "Stage & Screen---Soundtrack", + "Stage & Screen---Theme" + ], + "model_types": [ + "frozen_model", + "SavedModel", + "onnx" + ], + "dataset": { + "name": "Discogs-4M (unreleased)", + "citation": "In-house dataset", + "size": "4M full tracks (3.3M used)", + "metrics": { + "ROC-AUC": 0.95417, + "PR-AUC": 0.20629 + } + }, + "schema": { + "inputs": [ + { + "name": "serving_default_melspectrogram", + "type": "float", + "shape": [ + 64, + 128, + 96 + ] + } + ], + "outputs": [ + { + "name": "PartitionedCall:0", + "type": "float", + "shape": [ + 64, + 400 + ], + "op": "Sigmoid", + "output_purpose": "predictions" + }, + { + "name": "PartitionedCall:1", + "type": "float", + "shape": [ + 64, + 1280 + ], + "op": "Flatten", + "output_purpose": "embeddings" + } + ] + }, + "citation": "@inproceedings{alonso2022music,\n title={Music Representation Learning Based on Editorial Metadata from Discogs},\n author={Alonso-Jim{\\'e}nez, Pablo and Serra, Xavier and Bogdanov, Dmitry},\n booktitle={Conference of the International Society for Music Information Retrieval (ISMIR)},\n year={2022}\n}", + "inference": { + "sample_rate": 16000, + "algorithm": "TensorflowPredictEffnetDiscogs" + } +} \ No newline at end of file diff --git a/scripts/classificacao/discogs-effnet-bs64-1.pb b/scripts/classificacao/discogs-effnet-bs64-1.pb new file mode 100644 index 0000000..48b1257 Binary files /dev/null and b/scripts/classificacao/discogs-effnet-bs64-1.pb differ diff --git a/scripts/classificacao/genre_discogs400.json b/scripts/classificacao/genre_discogs400.json new file mode 100644 index 0000000..d86c817 --- /dev/null +++ b/scripts/classificacao/genre_discogs400.json @@ -0,0 +1,452 @@ +{ + "name": "Genre Discogs400", + "type": "Music genre classification", + "link": "https://essentia.upf.edu/models/classification-heads/genre_discogs400/genre_discogs400-discogs-effnet-1.pb", + "version": "1", + "description": "Prediction of 400 music styles in the from the Discogs taxonomy", + "author": "Pablo Alonso", + "email": "pablo.alonso@upf.edu", + "release_date": "2023-05-04", + "framework": "tensorflow", + "framework_version": "2.8.0", + "classes": [ + "Blues---Boogie Woogie", + "Blues---Chicago Blues", + "Blues---Country Blues", + "Blues---Delta Blues", + "Blues---Electric Blues", + "Blues---Harmonica Blues", + "Blues---Jump Blues", + "Blues---Louisiana Blues", + "Blues---Modern Electric Blues", + "Blues---Piano Blues", + "Blues---Rhythm & Blues", + "Blues---Texas Blues", + "Brass & Military---Brass Band", + "Brass & Military---Marches", + "Brass & Military---Military", + "Children's---Educational", + "Children's---Nursery Rhymes", + "Children's---Story", + "Classical---Baroque", + "Classical---Choral", + "Classical---Classical", + "Classical---Contemporary", + "Classical---Impressionist", + "Classical---Medieval", + "Classical---Modern", + "Classical---Neo-Classical", + "Classical---Neo-Romantic", + "Classical---Opera", + "Classical---Post-Modern", + "Classical---Renaissance", + "Classical---Romantic", + "Electronic---Abstract", + "Electronic---Acid", + "Electronic---Acid House", + "Electronic---Acid Jazz", + "Electronic---Ambient", + "Electronic---Bassline", + "Electronic---Beatdown", + "Electronic---Berlin-School", + "Electronic---Big Beat", + "Electronic---Bleep", + "Electronic---Breakbeat", + "Electronic---Breakcore", + "Electronic---Breaks", + "Electronic---Broken Beat", + "Electronic---Chillwave", + "Electronic---Chiptune", + "Electronic---Dance-pop", + "Electronic---Dark Ambient", + "Electronic---Darkwave", + "Electronic---Deep House", + "Electronic---Deep Techno", + "Electronic---Disco", + "Electronic---Disco Polo", + "Electronic---Donk", + "Electronic---Downtempo", + "Electronic---Drone", + "Electronic---Drum n Bass", + "Electronic---Dub", + "Electronic---Dub Techno", + "Electronic---Dubstep", + "Electronic---Dungeon Synth", + "Electronic---EBM", + "Electronic---Electro", + "Electronic---Electro House", + "Electronic---Electroclash", + "Electronic---Euro House", + "Electronic---Euro-Disco", + "Electronic---Eurobeat", + "Electronic---Eurodance", + "Electronic---Experimental", + "Electronic---Freestyle", + "Electronic---Future Jazz", + "Electronic---Gabber", + "Electronic---Garage House", + "Electronic---Ghetto", + "Electronic---Ghetto House", + "Electronic---Glitch", + "Electronic---Goa Trance", + "Electronic---Grime", + "Electronic---Halftime", + "Electronic---Hands Up", + "Electronic---Happy Hardcore", + "Electronic---Hard House", + "Electronic---Hard Techno", + "Electronic---Hard Trance", + "Electronic---Hardcore", + "Electronic---Hardstyle", + "Electronic---Hi NRG", + "Electronic---Hip Hop", + "Electronic---Hip-House", + "Electronic---House", + "Electronic---IDM", + "Electronic---Illbient", + "Electronic---Industrial", + "Electronic---Italo House", + "Electronic---Italo-Disco", + "Electronic---Italodance", + "Electronic---Jazzdance", + "Electronic---Juke", + "Electronic---Jumpstyle", + "Electronic---Jungle", + "Electronic---Latin", + "Electronic---Leftfield", + "Electronic---Makina", + "Electronic---Minimal", + "Electronic---Minimal Techno", + "Electronic---Modern Classical", + "Electronic---Musique Concr\u00e8te", + "Electronic---Neofolk", + "Electronic---New Age", + "Electronic---New Beat", + "Electronic---New Wave", + "Electronic---Noise", + "Electronic---Nu-Disco", + "Electronic---Power Electronics", + "Electronic---Progressive Breaks", + "Electronic---Progressive House", + "Electronic---Progressive Trance", + "Electronic---Psy-Trance", + "Electronic---Rhythmic Noise", + "Electronic---Schranz", + "Electronic---Sound Collage", + "Electronic---Speed Garage", + "Electronic---Speedcore", + "Electronic---Synth-pop", + "Electronic---Synthwave", + "Electronic---Tech House", + "Electronic---Tech Trance", + "Electronic---Techno", + "Electronic---Trance", + "Electronic---Tribal", + "Electronic---Tribal House", + "Electronic---Trip Hop", + "Electronic---Tropical House", + "Electronic---UK Garage", + "Electronic---Vaporwave", + "Folk, World, & Country---African", + "Folk, World, & Country---Bluegrass", + "Folk, World, & Country---Cajun", + "Folk, World, & Country---Canzone Napoletana", + "Folk, World, & Country---Catalan Music", + "Folk, World, & Country---Celtic", + "Folk, World, & Country---Country", + "Folk, World, & Country---Fado", + "Folk, World, & Country---Flamenco", + "Folk, World, & Country---Folk", + "Folk, World, & Country---Gospel", + "Folk, World, & Country---Highlife", + "Folk, World, & Country---Hillbilly", + "Folk, World, & Country---Hindustani", + "Folk, World, & Country---Honky Tonk", + "Folk, World, & Country---Indian Classical", + "Folk, World, & Country---La\u00efk\u00f3", + "Folk, World, & Country---Nordic", + "Folk, World, & Country---Pacific", + "Folk, World, & Country---Polka", + "Folk, World, & Country---Ra\u00ef", + "Folk, World, & Country---Romani", + "Folk, World, & Country---Soukous", + "Folk, World, & Country---S\u00e9ga", + "Folk, World, & Country---Volksmusik", + "Folk, World, & Country---Zouk", + "Folk, World, & Country---\u00c9ntekhno", + "Funk / Soul---Afrobeat", + "Funk / Soul---Boogie", + "Funk / Soul---Contemporary R&B", + "Funk / Soul---Disco", + "Funk / Soul---Free Funk", + "Funk / Soul---Funk", + "Funk / Soul---Gospel", + "Funk / Soul---Neo Soul", + "Funk / Soul---New Jack Swing", + "Funk / Soul---P.Funk", + "Funk / Soul---Psychedelic", + "Funk / Soul---Rhythm & Blues", + "Funk / Soul---Soul", + "Funk / Soul---Swingbeat", + "Funk / Soul---UK Street Soul", + "Hip Hop---Bass Music", + "Hip Hop---Boom Bap", + "Hip Hop---Bounce", + "Hip Hop---Britcore", + "Hip Hop---Cloud Rap", + "Hip Hop---Conscious", + "Hip Hop---Crunk", + "Hip Hop---Cut-up/DJ", + "Hip Hop---DJ Battle Tool", + "Hip Hop---Electro", + "Hip Hop---G-Funk", + "Hip Hop---Gangsta", + "Hip Hop---Grime", + "Hip Hop---Hardcore Hip-Hop", + "Hip Hop---Horrorcore", + "Hip Hop---Instrumental", + "Hip Hop---Jazzy Hip-Hop", + "Hip Hop---Miami Bass", + "Hip Hop---Pop Rap", + "Hip Hop---Ragga HipHop", + "Hip Hop---RnB/Swing", + "Hip Hop---Screw", + "Hip Hop---Thug Rap", + "Hip Hop---Trap", + "Hip Hop---Trip Hop", + "Hip Hop---Turntablism", + "Jazz---Afro-Cuban Jazz", + "Jazz---Afrobeat", + "Jazz---Avant-garde Jazz", + "Jazz---Big Band", + "Jazz---Bop", + "Jazz---Bossa Nova", + "Jazz---Contemporary Jazz", + "Jazz---Cool Jazz", + "Jazz---Dixieland", + "Jazz---Easy Listening", + "Jazz---Free Improvisation", + "Jazz---Free Jazz", + "Jazz---Fusion", + "Jazz---Gypsy Jazz", + "Jazz---Hard Bop", + "Jazz---Jazz-Funk", + "Jazz---Jazz-Rock", + "Jazz---Latin Jazz", + "Jazz---Modal", + "Jazz---Post Bop", + "Jazz---Ragtime", + "Jazz---Smooth Jazz", + "Jazz---Soul-Jazz", + "Jazz---Space-Age", + "Jazz---Swing", + "Latin---Afro-Cuban", + "Latin---Bai\u00e3o", + "Latin---Batucada", + "Latin---Beguine", + "Latin---Bolero", + "Latin---Boogaloo", + "Latin---Bossanova", + "Latin---Cha-Cha", + "Latin---Charanga", + "Latin---Compas", + "Latin---Cubano", + "Latin---Cumbia", + "Latin---Descarga", + "Latin---Forr\u00f3", + "Latin---Guaguanc\u00f3", + "Latin---Guajira", + "Latin---Guaracha", + "Latin---MPB", + "Latin---Mambo", + "Latin---Mariachi", + "Latin---Merengue", + "Latin---Norte\u00f1o", + "Latin---Nueva Cancion", + "Latin---Pachanga", + "Latin---Porro", + "Latin---Ranchera", + "Latin---Reggaeton", + "Latin---Rumba", + "Latin---Salsa", + "Latin---Samba", + "Latin---Son", + "Latin---Son Montuno", + "Latin---Tango", + "Latin---Tejano", + "Latin---Vallenato", + "Non-Music---Audiobook", + "Non-Music---Comedy", + "Non-Music---Dialogue", + "Non-Music---Education", + "Non-Music---Field Recording", + "Non-Music---Interview", + "Non-Music---Monolog", + "Non-Music---Poetry", + "Non-Music---Political", + "Non-Music---Promotional", + "Non-Music---Radioplay", + "Non-Music---Religious", + "Non-Music---Spoken Word", + "Pop---Ballad", + "Pop---Bollywood", + "Pop---Bubblegum", + "Pop---Chanson", + "Pop---City Pop", + "Pop---Europop", + "Pop---Indie Pop", + "Pop---J-pop", + "Pop---K-pop", + "Pop---Kay\u014dkyoku", + "Pop---Light Music", + "Pop---Music Hall", + "Pop---Novelty", + "Pop---Parody", + "Pop---Schlager", + "Pop---Vocal", + "Reggae---Calypso", + "Reggae---Dancehall", + "Reggae---Dub", + "Reggae---Lovers Rock", + "Reggae---Ragga", + "Reggae---Reggae", + "Reggae---Reggae-Pop", + "Reggae---Rocksteady", + "Reggae---Roots Reggae", + "Reggae---Ska", + "Reggae---Soca", + "Rock---AOR", + "Rock---Acid Rock", + "Rock---Acoustic", + "Rock---Alternative Rock", + "Rock---Arena Rock", + "Rock---Art Rock", + "Rock---Atmospheric Black Metal", + "Rock---Avantgarde", + "Rock---Beat", + "Rock---Black Metal", + "Rock---Blues Rock", + "Rock---Brit Pop", + "Rock---Classic Rock", + "Rock---Coldwave", + "Rock---Country Rock", + "Rock---Crust", + "Rock---Death Metal", + "Rock---Deathcore", + "Rock---Deathrock", + "Rock---Depressive Black Metal", + "Rock---Doo Wop", + "Rock---Doom Metal", + "Rock---Dream Pop", + "Rock---Emo", + "Rock---Ethereal", + "Rock---Experimental", + "Rock---Folk Metal", + "Rock---Folk Rock", + "Rock---Funeral Doom Metal", + "Rock---Funk Metal", + "Rock---Garage Rock", + "Rock---Glam", + "Rock---Goregrind", + "Rock---Goth Rock", + "Rock---Gothic Metal", + "Rock---Grindcore", + "Rock---Grunge", + "Rock---Hard Rock", + "Rock---Hardcore", + "Rock---Heavy Metal", + "Rock---Indie Rock", + "Rock---Industrial", + "Rock---Krautrock", + "Rock---Lo-Fi", + "Rock---Lounge", + "Rock---Math Rock", + "Rock---Melodic Death Metal", + "Rock---Melodic Hardcore", + "Rock---Metalcore", + "Rock---Mod", + "Rock---Neofolk", + "Rock---New Wave", + "Rock---No Wave", + "Rock---Noise", + "Rock---Noisecore", + "Rock---Nu Metal", + "Rock---Oi", + "Rock---Parody", + "Rock---Pop Punk", + "Rock---Pop Rock", + "Rock---Pornogrind", + "Rock---Post Rock", + "Rock---Post-Hardcore", + "Rock---Post-Metal", + "Rock---Post-Punk", + "Rock---Power Metal", + "Rock---Power Pop", + "Rock---Power Violence", + "Rock---Prog Rock", + "Rock---Progressive Metal", + "Rock---Psychedelic Rock", + "Rock---Psychobilly", + "Rock---Pub Rock", + "Rock---Punk", + "Rock---Rock & Roll", + "Rock---Rockabilly", + "Rock---Shoegaze", + "Rock---Ska", + "Rock---Sludge Metal", + "Rock---Soft Rock", + "Rock---Southern Rock", + "Rock---Space Rock", + "Rock---Speed Metal", + "Rock---Stoner Rock", + "Rock---Surf", + "Rock---Symphonic Rock", + "Rock---Technical Death Metal", + "Rock---Thrash", + "Rock---Twist", + "Rock---Viking Metal", + "Rock---Y\u00e9-Y\u00e9", + "Stage & Screen---Musical", + "Stage & Screen---Score", + "Stage & Screen---Soundtrack", + "Stage & Screen---Theme" + ], + "model_types": ["frozen_model", "SavedModel", "onnx"], + "dataset": { + "name": "Discogs-4M (unreleased)", + "citation": "In-house dataset", + "size": "4M full tracks (3.3M used)", + "metrics": { + "ROC-AUC": 0.95417, + "PR-AUC": 0.20629 + } + }, + "schema": { + "inputs": [ + { + "name": "serving_default_model_Placeholder", + "type": "float", + "shape": ["batch_size", 1280] + } + ], + "outputs": [ + { + "name": "PartitionedCall:0", + "type": "float", + "shape": ["batch_size", 400], + "op": "Sigmoid", + "output_purpose": "predictions" + } + ] + }, + "citation": "@inproceedings{alonso2022music,\n title={Music Representation Learning Based on Editorial Metadata from Discogs},\n author={Alonso-Jim{\\'e}nez, Pablo and Serra, Xavier and Bogdanov, Dmitry},\n booktitle={Conference of the International Society for Music Information Retrieval (ISMIR)},\n year={2022}\n}", + "inference": { + "sample_rate": 16000, + "algorithm": "TensorflowPredict2D", + "embedding_model": { + "algorithm": "TensorflowPredictEffnetDiscogs", + "model_name": "discogs-effnet-bs64-1", + "link": "https://essentia.upf.edu/models/music-style-classification/discogs-effnet/discogs-effnet-bs64-1.pb" + } + } +} diff --git a/scripts/classificacao/genre_discogs400.pb b/scripts/classificacao/genre_discogs400.pb new file mode 100644 index 0000000..901c5f8 Binary files /dev/null and b/scripts/classificacao/genre_discogs400.pb differ diff --git a/scripts/handler/benchmark.py b/scripts/handler/benchmark.py new file mode 100644 index 0000000..bdf2f4c --- /dev/null +++ b/scripts/handler/benchmark.py @@ -0,0 +1,339 @@ +# benchmark.py + +import os +import shutil +import subprocess +import multiprocessing +import logging +import time +import platform +import sys +import re +import unicodedata +import csv +import psutil +from datetime import datetime, timedelta + +# Importando apenas o necessário (sem sobrescrever as pastas do utils nativo no processamento global) +from file_parser import parse_mmp_file +from file_saver import save_to_json, save_to_yaml +from utils import SRC_MMPSEARCH, MMP_FOLDER + +# === ISOLAMENTO DE DIRETÓRIOS PARA O BENCHMARK === +# Tudo será gerado dentro de "benchmark_output" para não afetar o Jekyll +BENCHMARK_OUT = os.path.join(SRC_MMPSEARCH, "benchmark_output") +DATA_FOLDER = os.path.join(BENCHMARK_OUT, "data") +METADATA_FOLDER = os.path.join(BENCHMARK_OUT, "metadata") +WAV_FOLDER = os.path.join(BENCHMARK_OUT, "wav") +MMPZ_FOLDER = os.path.join(BENCHMARK_OUT, "mmpz") +LOG_FOLDER = os.path.join(BENCHMARK_OUT, "logs") +SAIDA_ANALISES = os.path.join(BENCHMARK_OUT, "analises") +LOTES_FOLDER = os.path.join(BENCHMARK_OUT, "lotes_yaml") + +# === CONFIGURAÇÕES GERAIS === +TIMEOUT_RENDER_SECONDS = 300 +MIN_RAM_FREE_MB = 500 + +global_counter = None + + +def slugify(value): + value = str(value) + value = ( + unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + ) + value = value.lower() + value = re.sub(r"[^\w\s-]", "", value) + value = re.sub(r"[-\s_]+", "-", value) + return value.strip("-_") + + +def get_cpu_safe_count(): + try: + count = multiprocessing.cpu_count() + return max(1, count - 1) + except: + return 1 + + +def init_worker(counter): + global global_counter + global_counter = counter + + +# === FUNÇÃO ORIGINAL DO SEU CÓDIGO (Com leve ajuste de segurança de arquivos) === +def process_single_file(args): + file_name, clean_slug, total_files = args + file_path = os.path.join(MMP_FOLDER, file_name) + pid = os.getpid() + + global global_counter + current_idx = 0 + if global_counter: + with global_counter.get_lock(): + global_counter.value += 1 + current_idx = global_counter.value + + start_time = time.time() + result = { + "success": False, + "file": file_name, + "slug": clean_slug, + "data": None, + "error": None, + "duration_s": 0.0, + "ram_mb": 0.0, + "file_size_mb": 0.0, + } + + try: + if os.path.exists(file_path): + result["file_size_mb"] = os.path.getsize(file_path) / (1024 * 1024) + + mem = psutil.virtual_memory() + if (mem.available / (1024 * 1024)) < MIN_RAM_FREE_MB: + result["error"] = "RAM Insuficiente" + return result + + original_base_name = os.path.splitext(file_name)[0] + ogg_name = clean_slug + ".ogg" + json_name = clean_slug + ".json" + yml_name = clean_slug + ".yml" + target_mmp_path = "" + + if file_name.endswith(".mmpz"): + destination_path = os.path.join(MMPZ_FOLDER, file_name) + # ALERTA DE BENCHMARK: Trocado move por copy2 para não destruir o dataset de testes! + if not os.path.exists(destination_path): + shutil.copy2(file_path, destination_path) + + mmp_temp_name = clean_slug + ".mmp" + output_mmp_path = os.path.join(MMP_FOLDER, mmp_temp_name) + abs_dest = os.path.abspath(destination_path) + abs_mmp_out = os.path.abspath(output_mmp_path) + + with open(abs_mmp_out, "w") as outfile: + subprocess.run( + ["lmms", "--dump", abs_dest], + stdout=outfile, + stderr=subprocess.PIPE, + check=True, + timeout=60, + env={"QT_QPA_PLATFORM": "offscreen", **os.environ}, + ) + target_mmp_path = output_mmp_path + elif file_name.endswith(".mmp"): + target_mmp_path = file_path + + # Renderização do Áudio + if os.path.exists(target_mmp_path): + abs_target_mmp = os.path.abspath(target_mmp_path) + abs_ogg_out = os.path.abspath(os.path.join(WAV_FOLDER, ogg_name)) + + # Aqui estava o errinho do "wav_cmd" que arrumamos na revisão! + ogg_cmd = ["lmms", "-r", abs_target_mmp, "-o", abs_ogg_out, "-f", "ogg"] + render_process = subprocess.run( + ogg_cmd, + check=False, + capture_output=True, + text=True, + timeout=TIMEOUT_RENDER_SECONDS, + env={"QT_QPA_PLATFORM": "offscreen", **os.environ}, + ) + if render_process.returncode != 0: + raise subprocess.CalledProcessError( + render_process.returncode, "lmms render", stderr="Erro LMMS" + ) + + # Parsing do XML + if os.path.exists(target_mmp_path): + mmp_data = parse_mmp_file(target_mmp_path) + if mmp_data: + mmp_data["file"] = clean_slug + mmp_data["original_title"] = original_base_name + save_to_json(mmp_data, os.path.join(METADATA_FOLDER, json_name)) + save_to_yaml(mmp_data, os.path.join(DATA_FOLDER, yml_name)) + result["success"] = True + result["data"] = mmp_data + + try: + process = psutil.Process(pid) + # rss = Resident Set Size (RAM real usada pelo processo) + result["ram_mb"] = process.memory_info().rss / (1024 * 1024) + except: + result["ram_mb"] = 0.0 + + except Exception as e: + result["error"] = str(e) + + result["duration_s"] = time.time() - start_time + return result + + +# === MOTOR DE BENCHMARK === +def execute_batch(limit_files, mode): + """ + Executa um lote de arquivos. + Se mode == 'sequencial', usa apenas 1 núcleo. + Se mode == 'paralelo', usa o safe count de núcleos. + """ + start_time_global = time.time() + + # Prepara pastas limpas para o teste + for folder in [ + MMPZ_FOLDER, + WAV_FOLDER, + METADATA_FOLDER, + DATA_FOLDER, + LOTES_FOLDER, + SAIDA_ANALISES, + ]: + os.makedirs(folder, exist_ok=True) + + all_files_raw = [f for f in os.listdir(MMP_FOLDER) if f.endswith((".mmp", ".mmpz"))] + if limit_files > 0: + all_files_raw = all_files_raw[:limit_files] + + total_files = len(all_files_raw) + if total_files == 0: + return 0, 0, 0, 0, 0 + + shared_counter = multiprocessing.Value("i", 0) + tasks = [] + + for idx, file_name in enumerate(all_files_raw): + slug = slugify(os.path.splitext(file_name)[0]) + if not slug: + slug = f"proj-{idx}" + tasks.append((file_name, slug, total_files)) + + num_cores = 1 if mode == "sequencial" else get_cpu_safe_count() + + print( + f"\n[{mode.upper()}] Processando {total_files} arquivos com {num_cores} núcleo(s)..." + ) + + # Execução + with multiprocessing.Pool( + processes=num_cores, initializer=init_worker, initargs=(shared_counter,) + ) as pool: + results = pool.map(process_single_file, tasks) + + # --- CÁLCULO DE MÉDIAS E SUCESSOS --- + sucessos = [r for r in results if r["success"]] + qtd_sucessos = len(sucessos) + + avg_time = ( + sum(r["duration_s"] for r in sucessos) / qtd_sucessos if qtd_sucessos > 0 else 0 + ) + avg_size = ( + sum(r["file_size_mb"] for r in sucessos) / qtd_sucessos + if qtd_sucessos > 0 + else 0 + ) + avg_ram = ( + sum(r["ram_mb"] for r in sucessos) / qtd_sucessos if qtd_sucessos > 0 else 0 + ) + + duration_total = time.time() - start_time_global + + # --- SALVAR RELATÓRIO DETALHADO POR LOTE --- + detalhado_csv = os.path.join( + BENCHMARK_OUT, f"auditoria_detalhada_{mode}_{limit_files}.csv" + ) + with open(detalhado_csv, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow( + ["Arquivo", "Tamanho_MB", "Tempo_s", "RAM_MB", "Status", "Erro"] + ) + for r in results: + status_str = "SUCESSO" if r["success"] else "FALHA" + writer.writerow( + [ + r["file"], + f"{r['file_size_mb']:.2f}", + f"{r['duration_s']:.2f}", + f"{r['ram_mb']:.2f}", + status_str, + r["error"] or "", + ] + ) + + print( + f"[{mode.upper()}] Concluído em {duration_total:.2f}s (Sucessos: {qtd_sucessos}/{total_files})" + ) + print( + f"[{mode.upper()}] Médias -> Tempo: {avg_time:.2f}s | Tamanho: {avg_size:.2f}MB | RAM: {avg_ram:.2f}MB" + ) + + return duration_total, qtd_sucessos, avg_time, avg_size, avg_ram + + +def run_benchmark(): + print("==================================================") + print(" INICIANDO TESTE DE SPEEDUP (SEQUENCIAL VS PARALELO)") + print("==================================================") + + test_sizes = [1, 10, 100, 1000] + modes = ["sequencial", "paralelo"] + + results_table = [] + + for size in test_sizes: + print(f"\n>>> INICIANDO BATERIA PARA {size} ARQUIVO(S) <<<") + row = {"Tamanho": size} + + for mode in modes: + duration, success_count, avg_time, avg_size, avg_ram = execute_batch( + size, mode + ) + row[f"Tempo_Total_{mode}_(s)"] = round(duration, 2) + row[f"Tempo_Medio_Proj_{mode}_(s)"] = round(avg_time, 2) + row[f"Tamanho_Medio_Proj_(MB)"] = round( + avg_size, 2 + ) # O tamanho é igual para ambos, mas guardamos + row[f"RAM_Media_{mode}_(MB)"] = round(avg_ram, 2) + row[f"Sucesso_{mode}"] = success_count + + # Calcula o Speedup (Tempo Total Sequencial / Tempo Total Paralelo) + speedup = ( + row["Tempo_Total_sequencial_(s)"] / row["Tempo_Total_paralelo_(s)"] + if row["Tempo_Total_paralelo_(s)"] > 0 + else 0 + ) + row["Speedup"] = round(speedup, 2) + results_table.append(row) + + print(f"--- Fim da bateria de {size}. Speedup Alcançado: {speedup:.2f}x ---") + + # Salva o resultado final num CSV consolidado + os.makedirs(BENCHMARK_OUT, exist_ok=True) + csv_file = os.path.join(BENCHMARK_OUT, "resultado_grafico_speedup.csv") + + # Lista de colunas para o CSV final + colunas = [ + "Tamanho", + "Speedup", + "Tempo_Total_sequencial_(s)", + "Tempo_Total_paralelo_(s)", + "Tempo_Medio_Proj_sequencial_(s)", + "Tempo_Medio_Proj_paralelo_(s)", + "RAM_Media_sequencial_(MB)", + "RAM_Media_paralelo_(MB)", + "Tamanho_Medio_Proj_(MB)", + "Sucesso_sequencial", + "Sucesso_paralelo", + ] + + with open(csv_file, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=colunas) + writer.writeheader() + writer.writerows(results_table) + + print("\n==================================================") + print(f"BENCHMARK FINALIZADO! Resultados salvos em:\n{BENCHMARK_OUT}") + print("==================================================") + + +if __name__ == "__main__": + run_benchmark() \ No newline at end of file diff --git a/scripts/handler/benchmark_output.log b/scripts/handler/benchmark_output.log new file mode 100644 index 0000000..68284cc --- /dev/null +++ b/scripts/handler/benchmark_output.log @@ -0,0 +1,53 @@ +nohup: ignoring input +================================================== + INICIANDO TESTE DE SPEEDUP (SEQUENCIAL VS PARALELO) +================================================== + +>>> INICIANDO BATERIA PARA 1 ARQUIVO(S) <<< + +[SEQUENCIAL] Processando 1 arquivos com 1 núcleo(s)... +[SEQUENCIAL] Concluído em 43.78s (Sucessos: 1/1) +[SEQUENCIAL] Médias -> Tempo: 43.69s | Tamanho: 0.54MB | RAM: 27.22MB + +[PARALELO] Processando 1 arquivos com 11 núcleo(s)... +[PARALELO] Concluído em 46.51s (Sucessos: 1/1) +[PARALELO] Médias -> Tempo: 46.48s | Tamanho: 0.54MB | RAM: 27.58MB +--- Fim da bateria de 1. Speedup Alcançado: 0.94x --- + +>>> INICIANDO BATERIA PARA 10 ARQUIVO(S) <<< + +[SEQUENCIAL] Processando 10 arquivos com 1 núcleo(s)... +Process ForkPoolWorker-13: +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/pool.py", line 131, in worker + put((job, i, result)) + File "/usr/lib/python3.12/multiprocessing/queues.py", line 399, in put + self._writer.send_bytes(obj) + File "/usr/lib/python3.12/multiprocessing/connection.py", line 200, in send_bytes + self._send_bytes(m[offset:offset + size]) + File "/usr/lib/python3.12/multiprocessing/connection.py", line 420, in _send_bytes + self._send(header) + File "/usr/lib/python3.12/multiprocessing/connection.py", line 384, in _send + n = write(self._handle, buf) + ^^^^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap + self.run() + File "/usr/lib/python3.12/multiprocessing/process.py", line 108, in run + self._target(*self._args, **self._kwargs) + File "/usr/lib/python3.12/multiprocessing/pool.py", line 136, in worker + put((job, i, (False, wrapped))) + File "/usr/lib/python3.12/multiprocessing/queues.py", line 399, in put + self._writer.send_bytes(obj) + File "/usr/lib/python3.12/multiprocessing/connection.py", line 200, in send_bytes + self._send_bytes(m[offset:offset + size]) + File "/usr/lib/python3.12/multiprocessing/connection.py", line 420, in _send_bytes + self._send(header) + File "/usr/lib/python3.12/multiprocessing/connection.py", line 384, in _send + n = write(self._handle, buf) + ^^^^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe diff --git a/scripts/handler/crawler.py b/scripts/handler/crawler.py new file mode 100644 index 0000000..b5a455d --- /dev/null +++ b/scripts/handler/crawler.py @@ -0,0 +1,230 @@ +import os +import time +import threading +import requests +import csv +import psutil +import re +import unicodedata +from datetime import datetime, timedelta +from bs4 import BeautifulSoup +from urllib.parse import urljoin, urlparse, parse_qs +from concurrent.futures import ThreadPoolExecutor, as_completed + +from utils import ( + SRC_MMPSEARCH, + MMP_FOLDER, +) + +# ================= CONFIGURAÇÕES PADRONIZADAS ================= +BASE_DIR = SRC_MMPSEARCH +PASTA_DESTINO = MMP_FOLDER # Pasta de entrada do pipeline +LOG_DIR = os.path.join(BASE_DIR, "logs/crawler") +ARQUIVO_AUDITORIA = os.path.join(LOG_DIR, "audit_crawler.csv") + +# Segurança de Hardware +MIN_RAM_FREE_MB = 500 # Pausa se tiver menos que 500MB livre +MAX_WORKERS_SAFE = 4 # I/O Bound não precisa de tantos workers quanto CPU, mas 4 é seguro + +HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Compatible; MMPResearchBot/1.0; +http://seu-site.com)' +} + +# Cria pastas +for p in [PASTA_DESTINO, LOG_DIR]: + os.makedirs(p, exist_ok=True) + +# ================= FUNÇÕES AUXILIARES ================= + +def slugify(value): + """Sanitização robusta de nomes de arquivo (Padrão do Pipeline)""" + value = str(value) + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') + value = value.lower() + value = re.sub(r'[^\w\s-]', '', value) + value = re.sub(r'[-\s_]+', '-', value) + return value.strip('-_') + +def verificar_memoria_segura(): + """Retorna True se é seguro continuar""" + try: + mem = psutil.virtual_memory() + return (mem.available / (1024 * 1024)) > MIN_RAM_FREE_MB + except: + return True + +# ================= CLASSE DE AUDITORIA ================= +class AuditoriaCrawler: + def __init__(self, arquivo_csv): + self.arquivo_csv = arquivo_csv + self.inicio_global = time.time() + self.lock = threading.Lock() + + # Cria CSV se não existir + if not os.path.exists(self.arquivo_csv): + with open(self.arquivo_csv, 'w', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow([ + "Timestamp", "Nome_Original", "Slug_Final", + "Tamanho_MB", "Tempo_DL_s", "Velocidade_KBps", + "Status", "URL_Origem" + ]) + + def registrar_download(self, nome_orig, slug, tamanho_bytes, tempo_s, url, status="SUCESSO"): + with self.lock: + try: + tamanho_mb = tamanho_bytes / (1024 * 1024) + velocidade = (tamanho_bytes / 1024) / tempo_s if tempo_s > 0 else 0 + ts = datetime.now().strftime("%H:%M:%S") + + with open(self.arquivo_csv, 'a', newline='', encoding='utf-8') as f: + writer = csv.writer(f) + writer.writerow([ + ts, nome_orig, slug, + f"{tamanho_mb:.2f}", f"{tempo_s:.2f}", + f"{velocidade:.2f}", status, url + ]) + + # Log Console Simplificado + print(f"[{ts}] {status}: {slug} ({tamanho_mb:.2f}MB @ {velocidade:.0f}KB/s)") + except Exception as e: + print(f"Erro ao auditar: {e}") + +# ================= WORKER ================= + +def baixar_projeto_worker(args): + url_detalhes, auditor = args + + # Check de Segurança (Throttle) + if not verificar_memoria_segura(): + time.sleep(5) # Espera RAM liberar + if not verificar_memoria_segura(): + return (False, "RAM Crítica - Download Abortado") + + try: + # 1. Pega página de detalhes + resp = requests.get(url_detalhes, headers=HEADERS, timeout=15) + if resp.status_code != 200: + auditor.registrar_download("N/A", "N/A", 0, 0, url_detalhes, f"HTTP_{resp.status_code}") + return (False, f"HTTP {resp.status_code}") + + soup = BeautifulSoup(resp.text, 'html.parser') + + # Lógica de encontrar link de download (Manteve a sua, que é boa) + link_tag = soup.find('a', href=lambda h: h and 'download_file.php' in h) + if not link_tag: + link_tag = soup.find('a', string='Download') + + if link_tag: + url_download = urljoin(url_detalhes, link_tag['href']) + + inicio_dl = time.time() + + # Request do Arquivo + with requests.get(url_download, headers=HEADERS, stream=True, timeout=60) as r_file: + r_file.raise_for_status() + + # --- DESCIFRAR NOME (COM SANITIZAÇÃO) --- + nome_bruto = "" + if "Content-Disposition" in r_file.headers: + cd = r_file.headers["Content-Disposition"] + if "filename=" in cd: + nome_bruto = cd.split("filename=")[1].strip('"') + + if not nome_bruto: + query = parse_qs(urlparse(url_download).query) + nome_bruto = query['name'][0] if 'name' in query else f"proj_{int(time.time())}.mmpz" + + # Aplica Slugify (Padrão do Pipeline) + nome_base, ext = os.path.splitext(nome_bruto) + if not ext: ext = ".mmpz" # Default seguro + slug = slugify(nome_base) + nome_final = f"{slug}{ext}" + + caminho_final = os.path.join(PASTA_DESTINO, nome_final) + + # Skip se já existe + if os.path.exists(caminho_final): + auditor.registrar_download(nome_bruto, slug, os.path.getsize(caminho_final), 0, url_detalhes, "SKIPPED_EXISTS") + return (True, "Já existe") + + # Gravação em chunks (Seguro para RAM) + bytes_downloaded = 0 + with open(caminho_final, 'wb') as f: + for chunk in r_file.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + bytes_downloaded += len(chunk) + + tempo_total = time.time() - inicio_dl + + # Auditoria de Sucesso + auditor.registrar_download(nome_bruto, slug, bytes_downloaded, tempo_total, url_detalhes, "SUCESSO") + return (True, "OK") + + else: + auditor.registrar_download("N/A", "N/A", 0, 0, url_detalhes, "LINK_NOT_FOUND") + return (False, "Link não encontrado") + + except Exception as e: + auditor.registrar_download("Erro", "Erro", 0, 0, url_detalhes, f"EXCEPT_{str(e)[:20]}") + return (False, str(e)) + +# ================= MAIN ================= + +def crawler_manager(url_base, max_paginas=2): + print(f"\n=== INICIANDO CRAWLER PADRONIZADO ===") + print(f"Destino: {PASTA_DESTINO}") + print(f"Auditoria: {ARQUIVO_AUDITORIA}") + + auditor = AuditoriaCrawler(ARQUIVO_AUDITORIA) + pagina_atual = 1 + total_sucesso = 0 + + # Executor seguro + with ThreadPoolExecutor(max_workers=MAX_WORKERS_SAFE) as executor: + while pagina_atual <= max_paginas: + print(f"\n--- Varrendo Página {pagina_atual} ---") + + operador = '&' if '?' in url_base else '?' + url_pag = f"{url_base}{operador}page={pagina_atual}" + + try: + # Pega a lista de projetos da página + r = requests.get(url_pag, headers=HEADERS) + soup = BeautifulSoup(r.text, 'html.parser') + # Filtro específico do LMMS Sharing Platform + links = soup.find_all('a', href=lambda h: h and '?action=show&file=' in h) + urls_unicas = sorted(list(set([l['href'] for l in links]))) + + if not urls_unicas: + print("Nenhum projeto encontrado nesta página. Encerrando.") + break + + # Submete tarefas + futures = [] + for u in urls_unicas: + url_completa = urljoin(url_base, u) + futures.append(executor.submit(baixar_projeto_worker, (url_completa, auditor))) + + # Processa resultados conforme chegam + for future in as_completed(futures): + ok, msg = future.result() + if ok: total_sucesso += 1 + + except Exception as e: + print(f"Erro na paginação: {e}") + + pagina_atual += 1 + time.sleep(1) # Respeito ao servidor + + tempo_total = time.time() - auditor.inicio_global + print(f"\n=== CRAWLER FINALIZADO ===") + print(f"Tempo: {timedelta(seconds=int(tempo_total))}") + print(f"Downloads com Sucesso: {total_sucesso}") + +if __name__ == "__main__": + # URL de exemplo (Projetos ordenados por data) + URL_ALVO = 'https://lmms.io/lsp/?action=browse&category=Projects&sort=date' + + crawler_manager(URL_ALVO, max_paginas=499) \ No newline at end of file diff --git a/scripts/login/__pycache__/login.cpython-312.pyc b/scripts/login/__pycache__/login.cpython-312.pyc new file mode 100644 index 0000000..5d92b25 Binary files /dev/null and b/scripts/login/__pycache__/login.cpython-312.pyc differ diff --git a/scripts/login/login.py b/scripts/login/login.py new file mode 100755 index 0000000..dc2b687 --- /dev/null +++ b/scripts/login/login.py @@ -0,0 +1,72 @@ +from flask import Flask, request, jsonify, session +from flask_sqlalchemy import SQLAlchemy +from flask_bcrypt import Bcrypt +from flask_login import LoginManager, UserMixin, login_user, login_required, logout_user, current_user + +app = Flask(__name__) +app.config['SECRET_KEY'] = '25de5592bf94c2ca18e27baa0ae2d4ee22a63012f32e1be719d31f530c215a387b9ec0c9d96be38e80a7ccdd859e04408facefff8fd9119e7f5a2d987d85abb7' # Troque isso! +app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///users_mmpSearch.db' # O arquivo do banco +db = SQLAlchemy(app) +bcrypt = Bcrypt(app) +login_manager = LoginManager(app) + +# --- Modelo do Banco de Dados --- +class User(UserMixin, db.Model): + id = db.Column(db.Integer, primary_key=True) + username = db.Column(db.String(150), unique=True, nullable=False) + password = db.Column(db.String(150), nullable=False) + +# Cria o banco na primeira execução +with app.app_context(): + db.create_all() + +@login_manager.user_loader +def load_user(user_id): + return User.query.get(int(user_id)) + +# --- Rotas --- + +@app.route('/api/register', methods=['POST']) +def register(): + data = request.json + hashed_password = bcrypt.generate_password_hash(data['password']).decode('utf-8') + new_user = User(username=data['username'], password=hashed_password) + try: + db.session.add(new_user) + db.session.commit() + return jsonify({"message": "Usuário criado com sucesso!"}), 201 + except: + return jsonify({"message": "Usuário já existe"}), 400 + +@app.route('/api/login', methods=['POST']) +def login(): + data = request.json + user = User.query.filter_by(username=data['username']).first() + if user and bcrypt.check_password_hash(user.password, data['password']): + login_user(user) + return jsonify({"message": "Login realizado", "user": user.username}), 200 + return jsonify({"message": "Credenciais inválidas"}), 401 + +@app.route('/api/logout', methods=['POST']) +@login_required +def logout(): + logout_user() + return jsonify({"message": "Logout realizado"}), 200 + +@app.route('/api/check_auth', methods=['GET']) +def check_auth(): + if current_user.is_authenticated: + return jsonify({"logged_in": True, "user": current_user.username}) + return jsonify({"logged_in": False}) + +# Rota para seus uploads (exemplo de como proteger) +@app.route('/api/upload_seguro', methods=['POST']) +@login_required +def upload_seguro(): + # Aqui entraria sua lógica de salvar o arquivo + # Como você já tem a lógica pronta, você pode chamar ela aqui + # ou mover o código para cá. + return jsonify({"message": f"Upload recebido de {current_user.username}"}) + +if __name__ == '__main__': + app.run(port=33005) \ No newline at end of file diff --git a/scripts/pipeline_etl.sh b/scripts/pipeline_etl.sh new file mode 100755 index 0000000..95223e3 --- /dev/null +++ b/scripts/pipeline_etl.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Interrompe a execução imediatamente se qualquer comando falhar +set -e + +# Define o caminho absoluto para o seu Python do ambiente virtual +PYTHON_BIN="/nethome/jotachina/projetos/mmpSearch/venv/bin/python3" + +# Define o diretório raiz do projeto para garantir que os caminhos relativos funcionem +BASE_DIR="/nethome/jotachina/projetos/mmpSearch/scripts" + +echo "=================================================================" +echo "Iniciando Pipeline Completo do MMPSearch" +echo "=================================================================" + +# Navega até a raiz do projeto para que os caminhos dos scripts funcionem corretamente +cd "$BASE_DIR" + +echo "[1/3] Executando: handler/main.py..." +"$PYTHON_BIN" handler/main.py +echo "handler/main.py concluído!" +echo "-----------------------------------------------------------------" + +echo "[2/3] Executando: classificacao/analise_audio.py..." +"$PYTHON_BIN" classificacao/analise_audio.py +echo "classificacao/analise_audio.py concluído!" +echo "-----------------------------------------------------------------" + +echo "[3/3] Executando: classificacao/classificacao_mestre.py..." +"$PYTHON_BIN" classificacao/classificacao_mestre.py +echo "classificacao/classificacao_mestre.py concluído!" +echo "=================================================================" +echo "Pipeline finalizado com sucesso!" +echo "=================================================================" \ No newline at end of file