Compare commits
	
		
			3 Commits
		
	
	
		
			4b52d4aa91
			...
			b5f235f5ae
		
	
	| Author | SHA1 | Date | 
|---|---|---|
|  | b5f235f5ae | |
|  | 387cb18bab | |
|  | f7e3913d2e | 
|  | @ -8,3 +8,41 @@ databases/storage.db | ||||||
| .vscode/* | .vscode/* | ||||||
| __pycache__/* | __pycache__/* | ||||||
| 
 | 
 | ||||||
|  | conf/experiment_config.json | ||||||
|  | example/Gmail.zip | ||||||
|  | example/Gmail/20240530_112812.jpg | ||||||
|  | example/Gmail/20240530_112825.jpg | ||||||
|  | example/Gmail/20240530_112925.jpg | ||||||
|  | example/Gmail/20240530_113110.jpg | ||||||
|  | example/Gmail/20240530_113114.jpg | ||||||
|  | example/Gmail/20240530_113531.jpg | ||||||
|  | example/Gmail/20240530_113555.jpg | ||||||
|  | example/Gmail/20240530_113614.jpg | ||||||
|  | example/Gmail/20240530_113650.jpg | ||||||
|  | example/Gmail/20240530_113731.jpg | ||||||
|  | example/audio/audio-file.flac | ||||||
|  | example/audio/audio-file2.flac | ||||||
|  | example/audio/AwACAgEAAxkBAAIBw2YX8o2vGGCNtZCXk7mY1Bm5w__lAAJmBAACxe7ARI1fUWAGcz_RNAQ.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAAIBxWYX8z-FgaOqQ5wJ4588Oa4wKNbsAAJnBAACxe7AREtzfoE98xYHNAQ.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAAIByGYX-NDzOEo-RH2YjEgG081ujEd-AAIQBAACS5vARIG7Bw8TVGr2NAQ.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANfZg2OE9Zvvn_7jJl7DaoD0j4kQQwAAqkEAAI-a3FEcjssrLJQjM00BA.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANhZg2U46T2Q6UYZ9gl3fHJsmJx2XcAAtgEAAI5ImlETOJIxvExjdg0BA.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANjZg2VEKqY04rTspcLw7trMmKPw58AAtkEAAI5ImlEZ18o0TpHffc0BA.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANlZg2WB8W9V-L0qkNENZrH1M1FZ5oAAtoEAAI5ImlEq70kYsaMeAs0BA.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANmZg2WDkyaygSa1QJvcSfYs6mNspcAAtsEAAI5ImlEfDhlNAY1KXs0BA.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANpZhcNbIqGT8cK3erus_8QoSBqkaMAAiEEAAIpsrhELqtOLaMMSf40BA.ogg | ||||||
|  | example/audio/AwACAgEAAxkBAANtZhcOLk3GkB7sTJ8HYUqjsmwjrNwAAiQEAAIpsrhEGvkyIIq7LKc0BA.ogg | ||||||
|  | example/audio/file_96.oga | ||||||
|  | example/audio/Prueba3.wav | ||||||
|  | example/audio/sin nombre.ogg | ||||||
|  | example/factura/Factura1.jpg | ||||||
|  | example/factura/Factura2.jpg | ||||||
|  | example/factura/Factura3.jpeg | ||||||
|  | example/factura/Factura4.jpg | ||||||
|  | example/factura/Factura5.jpg | ||||||
|  | example/factura/Factura6.jpg | ||||||
|  | example/texto/7fad5a297eb94ff9dbb3eff8e889a1c7dd87fe0f7c6bb6130f268acbfd828736.txt | ||||||
|  | 4b751a4425c2884286a92fde2de6427f_analitic_llm_rag.table | ||||||
|  | 4b751a4425c2884286a92fde2de6427f_analitic_llm_generaciontexto.table | ||||||
|  | 4b751a4425c2884286a92fde2de6427f_analitic_llm_factura.table | ||||||
|  | 4b751a4425c2884286a92fde2de6427f_analitic_llm_compra.table | ||||||
|  |  | ||||||
							
								
								
									
										290
									
								
								apis.py
								
								
								
								
							
							
						
						
									
										290
									
								
								apis.py
								
								
								
								
							|  | @ -1,10 +1,10 @@ | ||||||
| import fastapi | import fastapi | ||||||
| from fastapi import FastAPI, Request | from fastapi import FastAPI, Request | ||||||
| from fastapi.responses import HTMLResponse | from fastapi.responses import HTMLResponse,JSONResponse | ||||||
| from pydantic import BaseModel | from pydantic import BaseModel | ||||||
| import time | import time | ||||||
| from fastapi.staticfiles import StaticFiles | from fastapi.staticfiles import StaticFiles | ||||||
| from fastapi import FastAPI, Query, File, UploadFile | from fastapi import FastAPI, Query, File, UploadFile,HTTPException | ||||||
| #from fastapi.middleware.cors import CORSMiddleware | #from fastapi.middleware.cors import CORSMiddleware | ||||||
| from starlette.middleware.cors import CORSMiddleware | from starlette.middleware.cors import CORSMiddleware | ||||||
| import main | import main | ||||||
|  | @ -13,9 +13,23 @@ from databases import db | ||||||
| import audioread | import audioread | ||||||
| import pandas as pd | import pandas as pd | ||||||
| import statistics | import statistics | ||||||
|  | import hashlib | ||||||
|  | from datetime import datetime | ||||||
|  | import json | ||||||
| pwd = os.getcwd() | pwd = os.getcwd() | ||||||
| pathAud="example/audio" | pathAud="example/audio" | ||||||
| pathFact="example/factura" | pathFact="example/factura" | ||||||
|  | pathText="example/texto" | ||||||
|  | 
 | ||||||
|  | def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"): | ||||||
|  |     configPath=os.path.join(os.getcwd(),relPath) | ||||||
|  |     with open(configPath, 'r', encoding='utf-8') as file: | ||||||
|  |         config = json.load(file)[nameModel] | ||||||
|  |     Output= config[dataOut] | ||||||
|  |     return Output | ||||||
|  | mode_list=extractConfig(nameModel="SystemData",dataOut="mode_list") | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| app = FastAPI() | app = FastAPI() | ||||||
| #app.mount("/statics", StaticFiles(directory="statics"), name="statics") | #app.mount("/statics", StaticFiles(directory="statics"), name="statics") | ||||||
| app.add_middleware( | app.add_middleware( | ||||||
|  | @ -30,7 +44,6 @@ class Response(BaseModel): | ||||||
|     """Structure of data to querry of make post from X or article blog |     """Structure of data to querry of make post from X or article blog | ||||||
|     """ |     """ | ||||||
|     path: str = Query("", description="Style and sentiments of text") |     path: str = Query("", description="Style and sentiments of text") | ||||||
|     Trusted: str = Query("", description="Style and sentiments of text") |  | ||||||
|     model : str = Query("whisper", description="Style and sentiments of text") |     model : str = Query("whisper", description="Style and sentiments of text") | ||||||
| class Response1(BaseModel): | class Response1(BaseModel): | ||||||
|     path: str = Query("", description="path file") |     path: str = Query("", description="path file") | ||||||
|  | @ -55,29 +68,177 @@ class Response3(BaseModel): | ||||||
|     Trusted: str = Query("", description="Style and sentiments of text") |     Trusted: str = Query("", description="Style and sentiments of text") | ||||||
|     mode : str = Query("whisper", description="Style and sentiments of text") |     mode : str = Query("whisper", description="Style and sentiments of text") | ||||||
| 
 | 
 | ||||||
| 
 | #Funcionales | ||||||
| @app.get("/addTrusted") | @app.get("/addTrusted") | ||||||
| @app.post("/addTrusted") | @app.post("/addTrusted") | ||||||
| def addTrusted(response:Response3): | def addTrusted(response:Response3): | ||||||
|  |     """Api to add information of Trusted data | ||||||
|  | 
 | ||||||
|  |     Args: | ||||||
|  |         response (Response3): 3 params: | ||||||
|  |         path : path of archive on system if is a file OR text if is text. | ||||||
|  |         Trusted : information Trusted or better information in a process. | ||||||
|  |         mode: llm_compra,llm_factura,llm_generaciontexto,llm_rag,ocr,voice, | ||||||
|  | 
 | ||||||
|  |     Returns: | ||||||
|  |         _type_: _description_ | ||||||
|  |     """ | ||||||
|     path=response.path |     path=response.path | ||||||
|     Trusted=response.Trusted |     Trusted=response.Trusted | ||||||
|     mode=response.mode |     mode=response.mode | ||||||
|  |     last_modified=datetime.now() | ||||||
|  |     if mode not in mode_list.keys(): | ||||||
|  |         return JSONResponse( | ||||||
|  |         status_code=404, | ||||||
|  |         content={"content": "mode no found" } | ||||||
|  |     ) | ||||||
|  |     if mode == "llm_factura" or mode == "ocr" or mode == "voice": | ||||||
|  |         if not os.path.isfile(path): | ||||||
|  |             return JSONResponse( | ||||||
|  |             status_code=404, | ||||||
|  |             content={"content": "file no found" } | ||||||
|  |             ) | ||||||
|  |     if mode_list[mode]=="texto": | ||||||
|  |         hash1 = hashlib.sha256(path.encode()).hexdigest()+".txt" | ||||||
|  |         f = open("example/texto/"+hash1, "w") | ||||||
|  |         f.write(path) | ||||||
|  |         f.close() | ||||||
|  |         path=pwd+"/"+pathText+hash1 | ||||||
|  |         length=len(Trusted) | ||||||
|  |         size=0 | ||||||
|  |         duration=0 | ||||||
|  |     elif mode_list[mode]=="factura": | ||||||
|         file_stats = os.stat(path) |         file_stats = os.stat(path) | ||||||
|         size=file_stats.st_size / (1024 * 1024) |         size=file_stats.st_size / (1024 * 1024) | ||||||
|     if mode=="voice": |         length=0 | ||||||
|  |         duration=0 | ||||||
|  |     elif mode_list[mode]=="audio": | ||||||
|         with audioread.audio_open(path) as f:  |         with audioread.audio_open(path) as f:  | ||||||
|             duration = f.duration |             duration = f.duration | ||||||
|     else: |             length=0 | ||||||
|         duration = 0 |             size=0 | ||||||
|     if db(db.trusted.path == path and db.trusted.mode == mode).count()==0: | 
 | ||||||
|         db.trusted.insert(path=path,trusted=Trusted,mode=mode,size=size,duration =duration ) |     if db((db.trusted.path == path)&(db.trusted.mode == mode)).count()==0: | ||||||
|  |         db.trusted.insert(path=path,trusted=Trusted,mode=mode,size=size,duration=duration,last_modified=last_modified,length=length ) | ||||||
|         db.commit() |         db.commit() | ||||||
|         return "Add %s in mode %s"%(path,mode) |         return "Add %s in mode %s"%(path,mode) | ||||||
|     else: |     else: | ||||||
|         db(db.trusted.path == path and db.trusted.mode == mode).update(trusted=Trusted,size=size,duration =duration ) |         item=db((db.trusted.path == path)&(db.trusted.mode == mode)).select().last() | ||||||
|  |         modification_count=item.modification_count + 1 | ||||||
|  |         db((db.trusted.path == path)&(db.trusted.mode == mode)).update(trusted=Trusted,size=size,duration =duration,length=length,last_modified=last_modified,modification_count= modification_count) | ||||||
|         db.commit() |         db.commit() | ||||||
|         return "Update %s in mode %s"%(path,mode) |         return "Update %s in mode %s"%(path,mode) | ||||||
| 
 | 
 | ||||||
|  | @app.get("/EvalVoice") | ||||||
|  | @app.post("/EvalVoice") | ||||||
|  | def EvalVoice(response:Response): | ||||||
|  |     path=response.path | ||||||
|  |     model=response.model | ||||||
|  |     if db((db.trusted.path == path ) & ( db.trusted.mode == "voice")).count()==0: | ||||||
|  |         return JSONResponse( | ||||||
|  |         status_code=404, | ||||||
|  |         content={"content": "Trusted no found" } | ||||||
|  |     ) | ||||||
|  |      | ||||||
|  |     Trusted=db((db.trusted.path == path ) & ( db.trusted.mode == "voice")).select().last().trusted | ||||||
|  |     print(Trusted) | ||||||
|  |     if model=="whisper": | ||||||
|  |         Sal=main.EvalWhisper(path,Trusted) | ||||||
|  |     else: | ||||||
|  |         Sal=main.EvalVosk(path,Trusted) | ||||||
|  |     Sal["last_modified"]=datetime.now() | ||||||
|  |     if db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).count()==0: | ||||||
|  |         db.analitic_voice.insert(**Sal) | ||||||
|  |         db.commit() | ||||||
|  |     else: | ||||||
|  |         db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],last_modified=Sal["last_modified"]) | ||||||
|  |         db.commit() | ||||||
|  |     return Sal | ||||||
|  | 
 | ||||||
|  | @app.get("/evalvoicehtml") | ||||||
|  | def EvalVoicehtml(): | ||||||
|  |     dir_list = os.listdir(pathAud) | ||||||
|  |     Sal="" | ||||||
|  |     t=1 | ||||||
|  |     for i in dir_list: | ||||||
|  | 
 | ||||||
|  |         temp="""<option value="%s">Opción %s, %s</option> | ||||||
|  |         """%(str(pwd+"/"+pathAud+"/"+i),str(t),str(i)) | ||||||
|  |         Sal=Sal+temp | ||||||
|  |         t=t+1 | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  |     html="""<!DOCTYPE html> | ||||||
|  | <html lang="es"> | ||||||
|  | <head> | ||||||
|  |     <meta charset="UTF-8"> | ||||||
|  |     <meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||||||
|  |     <title>Evaluacion de modelos voice2txt</title> | ||||||
|  |     <style> | ||||||
|  |         body { | ||||||
|  |             font-family: Arial, sans-serif; | ||||||
|  |             margin: 20px; | ||||||
|  |         } | ||||||
|  |         input, button { | ||||||
|  |             margin: 10px 0; | ||||||
|  |             padding: 5px; | ||||||
|  |         } | ||||||
|  |         #respuesta { | ||||||
|  |             margin-top: 20px; | ||||||
|  |             padding: 10px; | ||||||
|  |             border: 1px solid #ccc; | ||||||
|  |             background-color: #f9f9f9; | ||||||
|  |         } | ||||||
|  |     </style> | ||||||
|  | </head> | ||||||
|  | <body> | ||||||
|  |     <h1>Petición POST a API</h1> | ||||||
|  | 
 | ||||||
|  |     <select id="texto1"> | ||||||
|  |         %s | ||||||
|  |     </select> | ||||||
|  |      | ||||||
|  |     <br> | ||||||
|  |     <select id="texto2"> | ||||||
|  |         <option value="whisper">whisper</option> | ||||||
|  |         <option value="vosk">vosk</option> | ||||||
|  |     </select> | ||||||
|  |     <br> | ||||||
|  |     <button onclick="enviarPeticion()">Enviar petición</button> | ||||||
|  |     <div id="respuesta"></div> | ||||||
|  | 
 | ||||||
|  |     <script> | ||||||
|  |         function enviarPeticion() { | ||||||
|  |             const texto1 = document.getElementById('texto1').value; | ||||||
|  |             const texto2 = document.getElementById('texto2').value; | ||||||
|  |             const datos = { | ||||||
|  |                 path: texto1, | ||||||
|  |                 model: texto2 | ||||||
|  |             }; | ||||||
|  | 
 | ||||||
|  |             fetch('/EvalVoice', { | ||||||
|  |                 method: 'POST', | ||||||
|  |                 headers: { | ||||||
|  |                     'Content-Type': 'application/json' | ||||||
|  |                 }, | ||||||
|  |                 body: JSON.stringify(datos) | ||||||
|  |             }) | ||||||
|  |             .then(response => response.json()) | ||||||
|  |             .then(data => { | ||||||
|  |                 document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2); | ||||||
|  |             }) | ||||||
|  |             .catch(error => { | ||||||
|  |                 document.getElementById('respuesta').innerHTML = 'Error: ' + error; | ||||||
|  |             }); | ||||||
|  |         } | ||||||
|  |     </script> | ||||||
|  | </body> | ||||||
|  | </html> | ||||||
|  |     """%(Sal) | ||||||
|  |     return HTMLResponse(content=html, status_code=200) | ||||||
|  | 
 | ||||||
|  | #Por revisar | ||||||
|  | 
 | ||||||
| def list2tablehtml(listdata,model): | def list2tablehtml(listdata,model): | ||||||
|     html="""<h2>Table of {0}</h2> |     html="""<h2>Table of {0}</h2> | ||||||
| <table style="width:100%"> | <table style="width:100%"> | ||||||
|  | @ -223,30 +384,6 @@ display:flex; | ||||||
|     return HTMLResponse(content=html, status_code=200) |     return HTMLResponse(content=html, status_code=200) | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| @app.get("/EvalVoice") |  | ||||||
| @app.post("/EvalVoice") |  | ||||||
| def EvalVoice(response:Response): |  | ||||||
|     path=response.path |  | ||||||
|     Trusted=response.Trusted |  | ||||||
|     model=response.model |  | ||||||
|     if Trusted=="": |  | ||||||
|         row=db(db.trusted.path == path and db.trusted.mode == "voice").select().first() |  | ||||||
|         try: |  | ||||||
|             Trusted=row.trusted |  | ||||||
|         except: |  | ||||||
|             pass |  | ||||||
| 
 |  | ||||||
|     if model=="whisper": |  | ||||||
|         Sal=main.EvalWhisper(path,Trusted) |  | ||||||
|     else: |  | ||||||
|         Sal=main.EvalVosk(path,Trusted) |  | ||||||
|     if db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).count()==0: |  | ||||||
|         db.analitic_voice.insert(**Sal) |  | ||||||
|         db.commit() |  | ||||||
|     else: |  | ||||||
|         db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"]) |  | ||||||
|         db.commit() |  | ||||||
|     return Sal |  | ||||||
| 
 | 
 | ||||||
| @app.get("/EvalFact") | @app.get("/EvalFact") | ||||||
| @app.post("/EvalFact") | @app.post("/EvalFact") | ||||||
|  | @ -290,91 +427,6 @@ def EvalLLMFact(response:Response2): | ||||||
|     Sal=main.EvalllmFacturas(path,task_prompt,system,content,max_tokens,model,prompt,TrustedLLmjson) |     Sal=main.EvalllmFacturas(path,task_prompt,system,content,max_tokens,model,prompt,TrustedLLmjson) | ||||||
|     return Sal |     return Sal | ||||||
| 
 | 
 | ||||||
| @app.get("/evalvoicehtml") |  | ||||||
| def EvalVoicehtml(): |  | ||||||
|     dir_list = os.listdir(pathAud) |  | ||||||
|     Sal="" |  | ||||||
|     t=1 |  | ||||||
|     for i in dir_list: |  | ||||||
| 
 |  | ||||||
|         temp="""<option value="%s">Opción %s, %s</option> |  | ||||||
|         """%(str(pwd+"/"+pathAud+"/"+i),str(t),str(i)) |  | ||||||
|         Sal=Sal+temp |  | ||||||
|         t=t+1 |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
|     html="""<!DOCTYPE html> |  | ||||||
| <html lang="es"> |  | ||||||
| <head> |  | ||||||
|     <meta charset="UTF-8"> |  | ||||||
|     <meta name="viewport" content="width=device-width, initial-scale=1.0"> |  | ||||||
|     <title>Evaluacion de modelos voice2txt</title> |  | ||||||
|     <style> |  | ||||||
|         body { |  | ||||||
|             font-family: Arial, sans-serif; |  | ||||||
|             margin: 20px; |  | ||||||
|         } |  | ||||||
|         input, button { |  | ||||||
|             margin: 10px 0; |  | ||||||
|             padding: 5px; |  | ||||||
|         } |  | ||||||
|         #respuesta { |  | ||||||
|             margin-top: 20px; |  | ||||||
|             padding: 10px; |  | ||||||
|             border: 1px solid #ccc; |  | ||||||
|             background-color: #f9f9f9; |  | ||||||
|         } |  | ||||||
|     </style> |  | ||||||
| </head> |  | ||||||
| <body> |  | ||||||
|     <h1>Petición POST a API</h1> |  | ||||||
| 
 |  | ||||||
|     <select id="texto1"> |  | ||||||
|         %s |  | ||||||
|     </select> |  | ||||||
|      |  | ||||||
|     <br> |  | ||||||
|     <input type="text" id="texto2" placeholder="Trusted"> |  | ||||||
|     <br> |  | ||||||
|     <select id="texto3"> |  | ||||||
|         <option value="whisper">whisper</option> |  | ||||||
|         <option value="vosk">vosk</option> |  | ||||||
|     </select> |  | ||||||
|     <br> |  | ||||||
|     <button onclick="enviarPeticion()">Enviar petición</button> |  | ||||||
|     <div id="respuesta"></div> |  | ||||||
| 
 |  | ||||||
|     <script> |  | ||||||
|         function enviarPeticion() { |  | ||||||
|             const texto1 = document.getElementById('texto1').value; |  | ||||||
|             const texto2 = document.getElementById('texto2').value; |  | ||||||
|             const texto3 = document.getElementById('texto3').value; |  | ||||||
|             const datos = { |  | ||||||
|                 path: texto1, |  | ||||||
|                 Trusted: texto2, |  | ||||||
|                 model: texto3 |  | ||||||
|             }; |  | ||||||
| 
 |  | ||||||
|             fetch('/EvalVoice', { |  | ||||||
|                 method: 'POST', |  | ||||||
|                 headers: { |  | ||||||
|                     'Content-Type': 'application/json' |  | ||||||
|                 }, |  | ||||||
|                 body: JSON.stringify(datos) |  | ||||||
|             }) |  | ||||||
|             .then(response => response.json()) |  | ||||||
|             .then(data => { |  | ||||||
|                 document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2); |  | ||||||
|             }) |  | ||||||
|             .catch(error => { |  | ||||||
|                 document.getElementById('respuesta').innerHTML = 'Error: ' + error; |  | ||||||
|             }); |  | ||||||
|         } |  | ||||||
|     </script> |  | ||||||
| </body> |  | ||||||
| </html> |  | ||||||
|     """%(Sal) |  | ||||||
|     return HTMLResponse(content=html, status_code=200) |  | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| @app.get("/evalocrfactura") | @app.get("/evalocrfactura") | ||||||
|  |  | ||||||
							
								
								
									
										54
									
								
								databases.py
								
								
								
								
							
							
						
						
									
										54
									
								
								databases.py
								
								
								
								
							|  | @ -5,8 +5,11 @@ db.define_table( | ||||||
|     Field("path"), |     Field("path"), | ||||||
|     Field("mode"), |     Field("mode"), | ||||||
|     Field("trusted"), |     Field("trusted"), | ||||||
|     Field("duration",type="double"), |     Field("duration",type="double",default=0),#audio | ||||||
|     Field("size",type="double") |     Field("sizeMB",type="double",default=0),# audio,factura | ||||||
|  |     Field("length",type="integer",default=0),#texto | ||||||
|  |     Field('last_modified', 'datetime'), | ||||||
|  |     Field('modification_count', 'integer', default=0) | ||||||
| ) | ) | ||||||
| db.define_table( | db.define_table( | ||||||
|     "analitic_voice", |     "analitic_voice", | ||||||
|  | @ -16,7 +19,8 @@ db.define_table( | ||||||
|     Field("time", type="double"), |     Field("time", type="double"), | ||||||
|     Field("path"), |     Field("path"), | ||||||
|     Field("similarity", type="double"), |     Field("similarity", type="double"), | ||||||
|     Field("similaritypartial", type="double")    |     Field("similaritypartial", type="double"), | ||||||
|  |     Field('last_modified', 'datetime')   | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| db.define_table( | db.define_table( | ||||||
|  | @ -28,16 +32,54 @@ db.define_table( | ||||||
|     Field("path"), |     Field("path"), | ||||||
|     Field("similarity", type="double"), |     Field("similarity", type="double"), | ||||||
|     Field("similaritypartial", type="double"), |     Field("similaritypartial", type="double"), | ||||||
|     Field("jsonok" ,type="integer")  |     Field("jsonok" ,type="integer"), | ||||||
|  |     Field('last_modified', 'datetime')   | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| db.define_table( | db.define_table( | ||||||
|     "analitic_llm", |     "analitic_llm_compra", | ||||||
|     Field("content"), |     Field("content"), | ||||||
|     Field("trusted"), |     Field("trusted"), | ||||||
|     Field("model"), |     Field("model"), | ||||||
|     Field("time", type="double"), |     Field("time", type="double"), | ||||||
|     Field("path"), |     Field("path"), | ||||||
|     Field("similarity", type="double"), |     Field("similarity", type="double"), | ||||||
|     Field("similaritypartial", type="double")    |     Field("similaritypartial", type="double"), | ||||||
|  |     Field('last_modified', 'datetime')     | ||||||
|  | ) | ||||||
|  | 
 | ||||||
|  | db.define_table( | ||||||
|  |     "analitic_llm_factura", | ||||||
|  |     Field("content"), | ||||||
|  |     Field("trusted"), | ||||||
|  |     Field("model"), | ||||||
|  |     Field("time", type="double"), | ||||||
|  |     Field("path"), | ||||||
|  |     Field("similarity", type="double"), | ||||||
|  |     Field("similaritypartial", type="double"), | ||||||
|  |     Field('last_modified', 'datetime')     | ||||||
|  | ) | ||||||
|  | 
 | ||||||
|  | db.define_table( | ||||||
|  |     "analitic_llm_generaciontexto", | ||||||
|  |     Field("content"), | ||||||
|  |     Field("trusted"), | ||||||
|  |     Field("model"), | ||||||
|  |     Field("time", type="double"), | ||||||
|  |     Field("path"), | ||||||
|  |     Field("similarity", type="double"), | ||||||
|  |     Field("similaritypartial", type="double"), | ||||||
|  |     Field('last_modified', 'datetime')     | ||||||
|  | ) | ||||||
|  | 
 | ||||||
|  | db.define_table( | ||||||
|  |     "analitic_llm_rag", | ||||||
|  |     Field("content"), | ||||||
|  |     Field("trusted"), | ||||||
|  |     Field("model"), | ||||||
|  |     Field("time", type="double"), | ||||||
|  |     Field("path"), | ||||||
|  |     Field("similarity", type="double"), | ||||||
|  |     Field("similaritypartial", type="double"), | ||||||
|  |     Field('last_modified', 'datetime')     | ||||||
| ) | ) | ||||||
|  | @ -0,0 +1,105 @@ | ||||||
|  | from taipy.gui import Gui | ||||||
|  | import hashlib | ||||||
|  | import json | ||||||
|  | import codecs, os | ||||||
|  | from taipy.gui import Html | ||||||
|  | import pandas as pd | ||||||
|  | import requests | ||||||
|  | import statistics | ||||||
|  | from databases import db | ||||||
|  | pwd = os.getcwd() | ||||||
|  | 
 | ||||||
|  | HTML = os.path.join(pwd,"html", "index.html") | ||||||
|  | file_read = codecs.open(HTML, "r", "utf-8") | ||||||
|  | index = file_read.read() | ||||||
|  | html_page_index = Html(index) | ||||||
|  | def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"): | ||||||
|  |     configPath=os.path.join(os.getcwd(),relPath) | ||||||
|  |     with open(configPath, 'r', encoding='utf-8') as file: | ||||||
|  |         config = json.load(file)[nameModel] | ||||||
|  |     Output= config[dataOut] | ||||||
|  |     return Output | ||||||
|  | mode_list=extractConfig(nameModel="SystemData",dataOut="mode_list") | ||||||
|  | def getmetricvoice(model): | ||||||
|  |     rows = db(db.analitic_voice.model==model).select() | ||||||
|  |     rows_list = rows.as_list() | ||||||
|  |     data=pd.DataFrame(rows_list) | ||||||
|  |     durationL=list() | ||||||
|  |     for i in rows_list: | ||||||
|  |         durationL.append(db(db.trusted.path == i["path"] ).select().last().duration) | ||||||
|  |     duration=statistics.mean(durationL) | ||||||
|  |     time=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['time'].values[0] | ||||||
|  |     similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0] | ||||||
|  |     similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0] | ||||||
|  |     efectivetime=time/duration | ||||||
|  |     return ({"model":model,"duration":duration,"time":time,"similarity":similarity,"similaritypartial":similaritypartial,"efectivetime":efectivetime}) | ||||||
|  | 
 | ||||||
|  | def html_getmetricvoice(): | ||||||
|  |     models=list() | ||||||
|  |     for row in db().select(db.analitic_voice.model, distinct=True): | ||||||
|  |         models.append(row.model) | ||||||
|  |     data={} | ||||||
|  |     for model in models: | ||||||
|  |         data[model]=getmetricvoice(model) | ||||||
|  |     data=pd.DataFrame(data).T | ||||||
|  |     datafiles={} | ||||||
|  |     for row in db().select(db.analitic_voice.ALL): | ||||||
|  |         datafiles[row.id]=row.as_dict() | ||||||
|  |     datafiles=pd.DataFrame(datafiles).T | ||||||
|  |     html=""" | ||||||
|  |     <taipy:table>{data_voice}</taipy:table> | ||||||
|  |     <taipy:table filter=True>{data_files_voice}</taipy:table> | ||||||
|  |     """ | ||||||
|  | 
 | ||||||
|  |     return html,data,datafiles | ||||||
|  | html_page_getmetricsvoice,data_voice,data_files_voices=html_getmetricvoice() | ||||||
|  | 
 | ||||||
|  | mode="voice" | ||||||
|  | modetypedata="audio" | ||||||
|  | file="id2" | ||||||
|  | def changemenu(mode): | ||||||
|  |     if mode_list[mode]=="audio": | ||||||
|  |         pathori="example/audio" | ||||||
|  |     if mode_list[mode]=="factura": | ||||||
|  |         pathori="example/factura" | ||||||
|  |     if mode_list[mode]=="texto": | ||||||
|  |         pathori="example/texto" | ||||||
|  |     seltypedata=mode_list[mode]   | ||||||
|  |     dir_list = os.listdir(pathori) | ||||||
|  |      | ||||||
|  |     return pathori,seltypedata,dir_list | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def trustedallhtml(mode): | ||||||
|  |     pathori,seltypedata,dir_list=changemenu(mode) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  |     textmode="" | ||||||
|  |     for modeused in mode_list.keys(): | ||||||
|  |         textmode=textmode+"('%s','%s'),"%(modeused,modeused) | ||||||
|  |     html="""<taipy:selector lov="{[%s]}" dropdown True on_change=changemenu>{sel}</taipy:selector>"""%(textmode)  | ||||||
|  |     Sal="" | ||||||
|  |     for i in dir_list: | ||||||
|  |         temp="""('%s', '%s'),"""%(str(pwd+"/"+pathori+"/"+i),str(i)) | ||||||
|  |         Sal=Sal+temp | ||||||
|  |     html2="""<taipy:selector lov="{[%s]}" dropdown True >{sel2}</taipy:selector>"""%(Sal) | ||||||
|  |     return html+html2 | ||||||
|  | 
 | ||||||
|  | html_page_trustedall = Html(trustedallhtml(mode)) | ||||||
|  | #print(sel,sel2,seltypedata) | ||||||
|  | HTML = os.path.join(pwd,"html", "index.html") | ||||||
|  | file_read = codecs.open(HTML, "r", "utf-8") | ||||||
|  | index = file_read.read() | ||||||
|  | html_page_index = Html(index) | ||||||
|  | 
 | ||||||
|  | data=pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]}) | ||||||
|  | 
 | ||||||
|  | pages = { | ||||||
|  |   "/": html_page_index , | ||||||
|  |   "getmetricsvoice": Html(html_page_getmetricsvoice), | ||||||
|  |   "trustedall":html_page_trustedall | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | app = Gui(pages=pages) | ||||||
|  | if __name__=="__main__": | ||||||
|  |     app.run(use_reloader=True,port=7882, change_delay=1600)#state.imageActive2, | ||||||
|  | @ -0,0 +1,22 @@ | ||||||
|  | <!DOCTYPE html> | ||||||
|  | <html lang="en"> | ||||||
|  | <head> | ||||||
|  |     <meta charset="UTF-8"> | ||||||
|  |     <meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||||||
|  |     <title>CIDITEL AI Playground</title> | ||||||
|  |     <link rel="stylesheet" tyle="text-decoration: none;" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css"> | ||||||
|  |     <link rel="stylesheet" tyle="text-decoration: none;" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@24,400,0,0" /> | ||||||
|  |     <link rel="shortcut icon" tyle="text-decoration: none;" href="statics/icons/favicon.svg" type="image/x-icon"> | ||||||
|  |     <link rel="stylesheet" tyle="text-decoration: none;" href="statics/css/style.css">   | ||||||
|  |     <link rel="stylesheet" tyle="text-decoration: none;" href="statics/css/media-queries.css">   | ||||||
|  |     <link rel="preconnect" tyle="text-decoration: none;" href="https://fonts.googleapis.com"> | ||||||
|  |     <link rel="preconnect" tyle="text-decoration: none;" href="https://fonts.gstatic.com" crossorigin> | ||||||
|  |     <link tyle="text-decoration: none;" href="https://fonts.googleapis.com/css2?family=Kanit:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&display=swap" rel="stylesheet"> | ||||||
|  | </head> | ||||||
|  | <body> | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  |     <taipy:table>{data}</taipy:table> | ||||||
|  | 
 | ||||||
|  | </body> | ||||||
|  | </html> | ||||||
|  | @ -0,0 +1,125 @@ | ||||||
|  | aiohttp==3.9.5 | ||||||
|  | aiosignal==1.3.1 | ||||||
|  | aniso8601==9.0.1 | ||||||
|  | annotated-types==0.7.0 | ||||||
|  | anyio==4.4.0 | ||||||
|  | apispec==6.4.0 | ||||||
|  | apispec-webframeworks==1.0.0 | ||||||
|  | arrow==1.3.0 | ||||||
|  | attrs==23.2.0 | ||||||
|  | audioread==3.0.1 | ||||||
|  | Automat==22.10.0 | ||||||
|  | bidict==0.23.1 | ||||||
|  | binaryornot==0.4.4 | ||||||
|  | blinker==1.8.2 | ||||||
|  | boto3==1.34.34 | ||||||
|  | botocore==1.34.150 | ||||||
|  | certifi==2024.2.2 | ||||||
|  | chardet==5.2.0 | ||||||
|  | charset-normalizer==3.3.2 | ||||||
|  | click==8.1.7 | ||||||
|  | constantly==23.10.4 | ||||||
|  | cookiecutter==2.5.0 | ||||||
|  | datasets==2.19.1 | ||||||
|  | deepdiff==6.7.1 | ||||||
|  | dill==0.3.8 | ||||||
|  | dnspython==2.6.1 | ||||||
|  | email_validator==2.2.0 | ||||||
|  | et-xmlfile==1.1.0 | ||||||
|  | evaluate==0.4.2 | ||||||
|  | fastapi==0.111.0 | ||||||
|  | fastapi-cli==0.0.4 | ||||||
|  | filelock==3.14.0 | ||||||
|  | Flask==3.0.2 | ||||||
|  | Flask-Cors==4.0.0 | ||||||
|  | Flask-RESTful==0.3.10 | ||||||
|  | Flask-SocketIO==5.3.6 | ||||||
|  | frozenlist==1.4.1 | ||||||
|  | fsspec==2024.3.1 | ||||||
|  | fuzzywuzzy==0.18.0 | ||||||
|  | gevent==23.9.1 | ||||||
|  | gevent-websocket==0.10.1 | ||||||
|  | gitignore_parser==0.1.11 | ||||||
|  | greenlet==3.0.3 | ||||||
|  | h11==0.14.0 | ||||||
|  | httpcore==1.0.5 | ||||||
|  | httptools==0.6.1 | ||||||
|  | httpx==0.27.0 | ||||||
|  | huggingface-hub==0.23.2 | ||||||
|  | hyperlink==21.0.0 | ||||||
|  | idna==3.7 | ||||||
|  | incremental==24.7.2 | ||||||
|  | itsdangerous==2.2.0 | ||||||
|  | Jinja2==3.1.4 | ||||||
|  | jmespath==1.0.1 | ||||||
|  | kthread==0.2.3 | ||||||
|  | Levenshtein==0.25.1 | ||||||
|  | Markdown==3.5.2 | ||||||
|  | markdown-it-py==3.0.0 | ||||||
|  | MarkupSafe==2.1.5 | ||||||
|  | marshmallow==3.20.2 | ||||||
|  | mdurl==0.1.2 | ||||||
|  | multidict==6.0.5 | ||||||
|  | multiprocess==0.70.16 | ||||||
|  | mutagen==1.47.0 | ||||||
|  | networkx==3.2.1 | ||||||
|  | numpy==1.26.4 | ||||||
|  | openpyxl==3.1.2 | ||||||
|  | ordered-set==4.1.0 | ||||||
|  | orjson==3.10.6 | ||||||
|  | packaging==24.0 | ||||||
|  | pandas==2.2.0 | ||||||
|  | passlib==1.7.4 | ||||||
|  | pyarrow==15.0.0 | ||||||
|  | pyarrow-hotfix==0.6 | ||||||
|  | pydal==20240713.1 | ||||||
|  | pydantic==2.8.2 | ||||||
|  | pydantic_core==2.20.1 | ||||||
|  | Pygments==2.18.0 | ||||||
|  | pymongo==4.6.1 | ||||||
|  | python-dateutil==2.9.0.post0 | ||||||
|  | python-dotenv==1.0.1 | ||||||
|  | python-engineio==4.9.1 | ||||||
|  | python-Levenshtein==0.25.1 | ||||||
|  | python-multipart==0.0.9 | ||||||
|  | python-slugify==8.0.4 | ||||||
|  | python-socketio==5.11.3 | ||||||
|  | pytz==2023.3.post1 | ||||||
|  | PyYAML==6.0.1 | ||||||
|  | rapidfuzz==3.9.4 | ||||||
|  | requests==2.32.3 | ||||||
|  | rich==13.7.1 | ||||||
|  | s3transfer==0.10.2 | ||||||
|  | shellingham==1.5.4 | ||||||
|  | simple-websocket==1.0.0 | ||||||
|  | six==1.16.0 | ||||||
|  | sniffio==1.3.1 | ||||||
|  | SQLAlchemy==2.0.25 | ||||||
|  | starlette==0.37.2 | ||||||
|  | taipy==3.1.1 | ||||||
|  | taipy-config==3.1.1 | ||||||
|  | taipy-core==3.1.1 | ||||||
|  | taipy-gui==3.1.4 | ||||||
|  | taipy-rest==3.1.1 | ||||||
|  | taipy-templates==3.1.1 | ||||||
|  | text-unidecode==1.3 | ||||||
|  | toml==0.10.2 | ||||||
|  | tqdm==4.66.4 | ||||||
|  | Twisted==23.10.0 | ||||||
|  | typer==0.12.3 | ||||||
|  | types-python-dateutil==2.9.0.20240316 | ||||||
|  | typing_extensions==4.12.0 | ||||||
|  | tzdata==2024.1 | ||||||
|  | tzlocal==5.2 | ||||||
|  | ujson==5.10.0 | ||||||
|  | urllib3==2.2.1 | ||||||
|  | uvicorn==0.30.1 | ||||||
|  | uvloop==0.19.0 | ||||||
|  | watchfiles==0.22.0 | ||||||
|  | websockets==12.0 | ||||||
|  | Werkzeug==3.0.3 | ||||||
|  | wsproto==1.2.0 | ||||||
|  | xxhash==3.4.1 | ||||||
|  | yarl==1.9.4 | ||||||
|  | zope.event==5.0 | ||||||
|  | zope.interface==6.4.post2 | ||||||
		Loading…
	
		Reference in New Issue