Compare commits
No commits in common. "b5f235f5ae849474a4460d1d409643a683f13b37" and "4b52d4aa91272d4b1c3bfd32b1a17cb5ab339b36" have entirely different histories.
b5f235f5ae
...
4b52d4aa91
|
|
@ -8,41 +8,3 @@ databases/storage.db
|
||||||
.vscode/*
|
.vscode/*
|
||||||
__pycache__/*
|
__pycache__/*
|
||||||
|
|
||||||
conf/experiment_config.json
|
|
||||||
example/Gmail.zip
|
|
||||||
example/Gmail/20240530_112812.jpg
|
|
||||||
example/Gmail/20240530_112825.jpg
|
|
||||||
example/Gmail/20240530_112925.jpg
|
|
||||||
example/Gmail/20240530_113110.jpg
|
|
||||||
example/Gmail/20240530_113114.jpg
|
|
||||||
example/Gmail/20240530_113531.jpg
|
|
||||||
example/Gmail/20240530_113555.jpg
|
|
||||||
example/Gmail/20240530_113614.jpg
|
|
||||||
example/Gmail/20240530_113650.jpg
|
|
||||||
example/Gmail/20240530_113731.jpg
|
|
||||||
example/audio/audio-file.flac
|
|
||||||
example/audio/audio-file2.flac
|
|
||||||
example/audio/AwACAgEAAxkBAAIBw2YX8o2vGGCNtZCXk7mY1Bm5w__lAAJmBAACxe7ARI1fUWAGcz_RNAQ.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAAIBxWYX8z-FgaOqQ5wJ4588Oa4wKNbsAAJnBAACxe7AREtzfoE98xYHNAQ.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAAIByGYX-NDzOEo-RH2YjEgG081ujEd-AAIQBAACS5vARIG7Bw8TVGr2NAQ.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANfZg2OE9Zvvn_7jJl7DaoD0j4kQQwAAqkEAAI-a3FEcjssrLJQjM00BA.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANhZg2U46T2Q6UYZ9gl3fHJsmJx2XcAAtgEAAI5ImlETOJIxvExjdg0BA.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANjZg2VEKqY04rTspcLw7trMmKPw58AAtkEAAI5ImlEZ18o0TpHffc0BA.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANlZg2WB8W9V-L0qkNENZrH1M1FZ5oAAtoEAAI5ImlEq70kYsaMeAs0BA.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANmZg2WDkyaygSa1QJvcSfYs6mNspcAAtsEAAI5ImlEfDhlNAY1KXs0BA.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANpZhcNbIqGT8cK3erus_8QoSBqkaMAAiEEAAIpsrhELqtOLaMMSf40BA.ogg
|
|
||||||
example/audio/AwACAgEAAxkBAANtZhcOLk3GkB7sTJ8HYUqjsmwjrNwAAiQEAAIpsrhEGvkyIIq7LKc0BA.ogg
|
|
||||||
example/audio/file_96.oga
|
|
||||||
example/audio/Prueba3.wav
|
|
||||||
example/audio/sin nombre.ogg
|
|
||||||
example/factura/Factura1.jpg
|
|
||||||
example/factura/Factura2.jpg
|
|
||||||
example/factura/Factura3.jpeg
|
|
||||||
example/factura/Factura4.jpg
|
|
||||||
example/factura/Factura5.jpg
|
|
||||||
example/factura/Factura6.jpg
|
|
||||||
example/texto/7fad5a297eb94ff9dbb3eff8e889a1c7dd87fe0f7c6bb6130f268acbfd828736.txt
|
|
||||||
4b751a4425c2884286a92fde2de6427f_analitic_llm_rag.table
|
|
||||||
4b751a4425c2884286a92fde2de6427f_analitic_llm_generaciontexto.table
|
|
||||||
4b751a4425c2884286a92fde2de6427f_analitic_llm_factura.table
|
|
||||||
4b751a4425c2884286a92fde2de6427f_analitic_llm_compra.table
|
|
||||||
|
|
|
||||||
294
apis.py
294
apis.py
|
|
@ -1,10 +1,10 @@
|
||||||
import fastapi
|
import fastapi
|
||||||
from fastapi import FastAPI, Request
|
from fastapi import FastAPI, Request
|
||||||
from fastapi.responses import HTMLResponse,JSONResponse
|
from fastapi.responses import HTMLResponse
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
import time
|
import time
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from fastapi import FastAPI, Query, File, UploadFile,HTTPException
|
from fastapi import FastAPI, Query, File, UploadFile
|
||||||
#from fastapi.middleware.cors import CORSMiddleware
|
#from fastapi.middleware.cors import CORSMiddleware
|
||||||
from starlette.middleware.cors import CORSMiddleware
|
from starlette.middleware.cors import CORSMiddleware
|
||||||
import main
|
import main
|
||||||
|
|
@ -13,23 +13,9 @@ from databases import db
|
||||||
import audioread
|
import audioread
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import statistics
|
import statistics
|
||||||
import hashlib
|
|
||||||
from datetime import datetime
|
|
||||||
import json
|
|
||||||
pwd = os.getcwd()
|
pwd = os.getcwd()
|
||||||
pathAud="example/audio"
|
pathAud="example/audio"
|
||||||
pathFact="example/factura"
|
pathFact="example/factura"
|
||||||
pathText="example/texto"
|
|
||||||
|
|
||||||
def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"):
|
|
||||||
configPath=os.path.join(os.getcwd(),relPath)
|
|
||||||
with open(configPath, 'r', encoding='utf-8') as file:
|
|
||||||
config = json.load(file)[nameModel]
|
|
||||||
Output= config[dataOut]
|
|
||||||
return Output
|
|
||||||
mode_list=extractConfig(nameModel="SystemData",dataOut="mode_list")
|
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
#app.mount("/statics", StaticFiles(directory="statics"), name="statics")
|
#app.mount("/statics", StaticFiles(directory="statics"), name="statics")
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
|
|
@ -44,6 +30,7 @@ class Response(BaseModel):
|
||||||
"""Structure of data to querry of make post from X or article blog
|
"""Structure of data to querry of make post from X or article blog
|
||||||
"""
|
"""
|
||||||
path: str = Query("", description="Style and sentiments of text")
|
path: str = Query("", description="Style and sentiments of text")
|
||||||
|
Trusted: str = Query("", description="Style and sentiments of text")
|
||||||
model : str = Query("whisper", description="Style and sentiments of text")
|
model : str = Query("whisper", description="Style and sentiments of text")
|
||||||
class Response1(BaseModel):
|
class Response1(BaseModel):
|
||||||
path: str = Query("", description="path file")
|
path: str = Query("", description="path file")
|
||||||
|
|
@ -68,177 +55,29 @@ class Response3(BaseModel):
|
||||||
Trusted: str = Query("", description="Style and sentiments of text")
|
Trusted: str = Query("", description="Style and sentiments of text")
|
||||||
mode : str = Query("whisper", description="Style and sentiments of text")
|
mode : str = Query("whisper", description="Style and sentiments of text")
|
||||||
|
|
||||||
#Funcionales
|
|
||||||
@app.get("/addTrusted")
|
@app.get("/addTrusted")
|
||||||
@app.post("/addTrusted")
|
@app.post("/addTrusted")
|
||||||
def addTrusted(response:Response3):
|
def addTrusted(response:Response3):
|
||||||
"""Api to add information of Trusted data
|
|
||||||
|
|
||||||
Args:
|
|
||||||
response (Response3): 3 params:
|
|
||||||
path : path of archive on system if is a file OR text if is text.
|
|
||||||
Trusted : information Trusted or better information in a process.
|
|
||||||
mode: llm_compra,llm_factura,llm_generaciontexto,llm_rag,ocr,voice,
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
_type_: _description_
|
|
||||||
"""
|
|
||||||
path=response.path
|
path=response.path
|
||||||
Trusted=response.Trusted
|
Trusted=response.Trusted
|
||||||
mode=response.mode
|
mode=response.mode
|
||||||
last_modified=datetime.now()
|
file_stats = os.stat(path)
|
||||||
if mode not in mode_list.keys():
|
size=file_stats.st_size / (1024 * 1024)
|
||||||
return JSONResponse(
|
if mode=="voice":
|
||||||
status_code=404,
|
|
||||||
content={"content": "mode no found" }
|
|
||||||
)
|
|
||||||
if mode == "llm_factura" or mode == "ocr" or mode == "voice":
|
|
||||||
if not os.path.isfile(path):
|
|
||||||
return JSONResponse(
|
|
||||||
status_code=404,
|
|
||||||
content={"content": "file no found" }
|
|
||||||
)
|
|
||||||
if mode_list[mode]=="texto":
|
|
||||||
hash1 = hashlib.sha256(path.encode()).hexdigest()+".txt"
|
|
||||||
f = open("example/texto/"+hash1, "w")
|
|
||||||
f.write(path)
|
|
||||||
f.close()
|
|
||||||
path=pwd+"/"+pathText+hash1
|
|
||||||
length=len(Trusted)
|
|
||||||
size=0
|
|
||||||
duration=0
|
|
||||||
elif mode_list[mode]=="factura":
|
|
||||||
file_stats = os.stat(path)
|
|
||||||
size=file_stats.st_size / (1024 * 1024)
|
|
||||||
length=0
|
|
||||||
duration=0
|
|
||||||
elif mode_list[mode]=="audio":
|
|
||||||
with audioread.audio_open(path) as f:
|
with audioread.audio_open(path) as f:
|
||||||
duration = f.duration
|
duration = f.duration
|
||||||
length=0
|
else:
|
||||||
size=0
|
duration = 0
|
||||||
|
if db(db.trusted.path == path and db.trusted.mode == mode).count()==0:
|
||||||
if db((db.trusted.path == path)&(db.trusted.mode == mode)).count()==0:
|
db.trusted.insert(path=path,trusted=Trusted,mode=mode,size=size,duration =duration )
|
||||||
db.trusted.insert(path=path,trusted=Trusted,mode=mode,size=size,duration=duration,last_modified=last_modified,length=length )
|
|
||||||
db.commit()
|
db.commit()
|
||||||
return "Add %s in mode %s"%(path,mode)
|
return "Add %s in mode %s"%(path,mode)
|
||||||
else:
|
else:
|
||||||
item=db((db.trusted.path == path)&(db.trusted.mode == mode)).select().last()
|
db(db.trusted.path == path and db.trusted.mode == mode).update(trusted=Trusted,size=size,duration =duration )
|
||||||
modification_count=item.modification_count + 1
|
|
||||||
db((db.trusted.path == path)&(db.trusted.mode == mode)).update(trusted=Trusted,size=size,duration =duration,length=length,last_modified=last_modified,modification_count= modification_count)
|
|
||||||
db.commit()
|
db.commit()
|
||||||
return "Update %s in mode %s"%(path,mode)
|
return "Update %s in mode %s"%(path,mode)
|
||||||
|
|
||||||
@app.get("/EvalVoice")
|
|
||||||
@app.post("/EvalVoice")
|
|
||||||
def EvalVoice(response:Response):
|
|
||||||
path=response.path
|
|
||||||
model=response.model
|
|
||||||
if db((db.trusted.path == path ) & ( db.trusted.mode == "voice")).count()==0:
|
|
||||||
return JSONResponse(
|
|
||||||
status_code=404,
|
|
||||||
content={"content": "Trusted no found" }
|
|
||||||
)
|
|
||||||
|
|
||||||
Trusted=db((db.trusted.path == path ) & ( db.trusted.mode == "voice")).select().last().trusted
|
|
||||||
print(Trusted)
|
|
||||||
if model=="whisper":
|
|
||||||
Sal=main.EvalWhisper(path,Trusted)
|
|
||||||
else:
|
|
||||||
Sal=main.EvalVosk(path,Trusted)
|
|
||||||
Sal["last_modified"]=datetime.now()
|
|
||||||
if db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).count()==0:
|
|
||||||
db.analitic_voice.insert(**Sal)
|
|
||||||
db.commit()
|
|
||||||
else:
|
|
||||||
db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],last_modified=Sal["last_modified"])
|
|
||||||
db.commit()
|
|
||||||
return Sal
|
|
||||||
|
|
||||||
@app.get("/evalvoicehtml")
|
|
||||||
def EvalVoicehtml():
|
|
||||||
dir_list = os.listdir(pathAud)
|
|
||||||
Sal=""
|
|
||||||
t=1
|
|
||||||
for i in dir_list:
|
|
||||||
|
|
||||||
temp="""<option value="%s">Opción %s, %s</option>
|
|
||||||
"""%(str(pwd+"/"+pathAud+"/"+i),str(t),str(i))
|
|
||||||
Sal=Sal+temp
|
|
||||||
t=t+1
|
|
||||||
|
|
||||||
|
|
||||||
html="""<!DOCTYPE html>
|
|
||||||
<html lang="es">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<title>Evaluacion de modelos voice2txt</title>
|
|
||||||
<style>
|
|
||||||
body {
|
|
||||||
font-family: Arial, sans-serif;
|
|
||||||
margin: 20px;
|
|
||||||
}
|
|
||||||
input, button {
|
|
||||||
margin: 10px 0;
|
|
||||||
padding: 5px;
|
|
||||||
}
|
|
||||||
#respuesta {
|
|
||||||
margin-top: 20px;
|
|
||||||
padding: 10px;
|
|
||||||
border: 1px solid #ccc;
|
|
||||||
background-color: #f9f9f9;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>Petición POST a API</h1>
|
|
||||||
|
|
||||||
<select id="texto1">
|
|
||||||
%s
|
|
||||||
</select>
|
|
||||||
|
|
||||||
<br>
|
|
||||||
<select id="texto2">
|
|
||||||
<option value="whisper">whisper</option>
|
|
||||||
<option value="vosk">vosk</option>
|
|
||||||
</select>
|
|
||||||
<br>
|
|
||||||
<button onclick="enviarPeticion()">Enviar petición</button>
|
|
||||||
<div id="respuesta"></div>
|
|
||||||
|
|
||||||
<script>
|
|
||||||
function enviarPeticion() {
|
|
||||||
const texto1 = document.getElementById('texto1').value;
|
|
||||||
const texto2 = document.getElementById('texto2').value;
|
|
||||||
const datos = {
|
|
||||||
path: texto1,
|
|
||||||
model: texto2
|
|
||||||
};
|
|
||||||
|
|
||||||
fetch('/EvalVoice', {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
},
|
|
||||||
body: JSON.stringify(datos)
|
|
||||||
})
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(data => {
|
|
||||||
document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
document.getElementById('respuesta').innerHTML = 'Error: ' + error;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
"""%(Sal)
|
|
||||||
return HTMLResponse(content=html, status_code=200)
|
|
||||||
|
|
||||||
#Por revisar
|
|
||||||
|
|
||||||
def list2tablehtml(listdata,model):
|
def list2tablehtml(listdata,model):
|
||||||
html="""<h2>Table of {0}</h2>
|
html="""<h2>Table of {0}</h2>
|
||||||
<table style="width:100%">
|
<table style="width:100%">
|
||||||
|
|
@ -384,6 +223,30 @@ display:flex;
|
||||||
return HTMLResponse(content=html, status_code=200)
|
return HTMLResponse(content=html, status_code=200)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/EvalVoice")
|
||||||
|
@app.post("/EvalVoice")
|
||||||
|
def EvalVoice(response:Response):
|
||||||
|
path=response.path
|
||||||
|
Trusted=response.Trusted
|
||||||
|
model=response.model
|
||||||
|
if Trusted=="":
|
||||||
|
row=db(db.trusted.path == path and db.trusted.mode == "voice").select().first()
|
||||||
|
try:
|
||||||
|
Trusted=row.trusted
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if model=="whisper":
|
||||||
|
Sal=main.EvalWhisper(path,Trusted)
|
||||||
|
else:
|
||||||
|
Sal=main.EvalVosk(path,Trusted)
|
||||||
|
if db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).count()==0:
|
||||||
|
db.analitic_voice.insert(**Sal)
|
||||||
|
db.commit()
|
||||||
|
else:
|
||||||
|
db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"])
|
||||||
|
db.commit()
|
||||||
|
return Sal
|
||||||
|
|
||||||
@app.get("/EvalFact")
|
@app.get("/EvalFact")
|
||||||
@app.post("/EvalFact")
|
@app.post("/EvalFact")
|
||||||
|
|
@ -427,6 +290,91 @@ def EvalLLMFact(response:Response2):
|
||||||
Sal=main.EvalllmFacturas(path,task_prompt,system,content,max_tokens,model,prompt,TrustedLLmjson)
|
Sal=main.EvalllmFacturas(path,task_prompt,system,content,max_tokens,model,prompt,TrustedLLmjson)
|
||||||
return Sal
|
return Sal
|
||||||
|
|
||||||
|
@app.get("/evalvoicehtml")
|
||||||
|
def EvalVoicehtml():
|
||||||
|
dir_list = os.listdir(pathAud)
|
||||||
|
Sal=""
|
||||||
|
t=1
|
||||||
|
for i in dir_list:
|
||||||
|
|
||||||
|
temp="""<option value="%s">Opción %s, %s</option>
|
||||||
|
"""%(str(pwd+"/"+pathAud+"/"+i),str(t),str(i))
|
||||||
|
Sal=Sal+temp
|
||||||
|
t=t+1
|
||||||
|
|
||||||
|
|
||||||
|
html="""<!DOCTYPE html>
|
||||||
|
<html lang="es">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Evaluacion de modelos voice2txt</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-family: Arial, sans-serif;
|
||||||
|
margin: 20px;
|
||||||
|
}
|
||||||
|
input, button {
|
||||||
|
margin: 10px 0;
|
||||||
|
padding: 5px;
|
||||||
|
}
|
||||||
|
#respuesta {
|
||||||
|
margin-top: 20px;
|
||||||
|
padding: 10px;
|
||||||
|
border: 1px solid #ccc;
|
||||||
|
background-color: #f9f9f9;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>Petición POST a API</h1>
|
||||||
|
|
||||||
|
<select id="texto1">
|
||||||
|
%s
|
||||||
|
</select>
|
||||||
|
|
||||||
|
<br>
|
||||||
|
<input type="text" id="texto2" placeholder="Trusted">
|
||||||
|
<br>
|
||||||
|
<select id="texto3">
|
||||||
|
<option value="whisper">whisper</option>
|
||||||
|
<option value="vosk">vosk</option>
|
||||||
|
</select>
|
||||||
|
<br>
|
||||||
|
<button onclick="enviarPeticion()">Enviar petición</button>
|
||||||
|
<div id="respuesta"></div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
function enviarPeticion() {
|
||||||
|
const texto1 = document.getElementById('texto1').value;
|
||||||
|
const texto2 = document.getElementById('texto2').value;
|
||||||
|
const texto3 = document.getElementById('texto3').value;
|
||||||
|
const datos = {
|
||||||
|
path: texto1,
|
||||||
|
Trusted: texto2,
|
||||||
|
model: texto3
|
||||||
|
};
|
||||||
|
|
||||||
|
fetch('/EvalVoice', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify(datos)
|
||||||
|
})
|
||||||
|
.then(response => response.json())
|
||||||
|
.then(data => {
|
||||||
|
document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
document.getElementById('respuesta').innerHTML = 'Error: ' + error;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""%(Sal)
|
||||||
|
return HTMLResponse(content=html, status_code=200)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/evalocrfactura")
|
@app.get("/evalocrfactura")
|
||||||
|
|
|
||||||
54
databases.py
54
databases.py
|
|
@ -5,11 +5,8 @@ db.define_table(
|
||||||
Field("path"),
|
Field("path"),
|
||||||
Field("mode"),
|
Field("mode"),
|
||||||
Field("trusted"),
|
Field("trusted"),
|
||||||
Field("duration",type="double",default=0),#audio
|
Field("duration",type="double"),
|
||||||
Field("sizeMB",type="double",default=0),# audio,factura
|
Field("size",type="double")
|
||||||
Field("length",type="integer",default=0),#texto
|
|
||||||
Field('last_modified', 'datetime'),
|
|
||||||
Field('modification_count', 'integer', default=0)
|
|
||||||
)
|
)
|
||||||
db.define_table(
|
db.define_table(
|
||||||
"analitic_voice",
|
"analitic_voice",
|
||||||
|
|
@ -19,8 +16,7 @@ db.define_table(
|
||||||
Field("time", type="double"),
|
Field("time", type="double"),
|
||||||
Field("path"),
|
Field("path"),
|
||||||
Field("similarity", type="double"),
|
Field("similarity", type="double"),
|
||||||
Field("similaritypartial", type="double"),
|
Field("similaritypartial", type="double")
|
||||||
Field('last_modified', 'datetime')
|
|
||||||
)
|
)
|
||||||
|
|
||||||
db.define_table(
|
db.define_table(
|
||||||
|
|
@ -32,54 +28,16 @@ db.define_table(
|
||||||
Field("path"),
|
Field("path"),
|
||||||
Field("similarity", type="double"),
|
Field("similarity", type="double"),
|
||||||
Field("similaritypartial", type="double"),
|
Field("similaritypartial", type="double"),
|
||||||
Field("jsonok" ,type="integer"),
|
Field("jsonok" ,type="integer")
|
||||||
Field('last_modified', 'datetime')
|
|
||||||
)
|
)
|
||||||
|
|
||||||
db.define_table(
|
db.define_table(
|
||||||
"analitic_llm_compra",
|
"analitic_llm",
|
||||||
Field("content"),
|
Field("content"),
|
||||||
Field("trusted"),
|
Field("trusted"),
|
||||||
Field("model"),
|
Field("model"),
|
||||||
Field("time", type="double"),
|
Field("time", type="double"),
|
||||||
Field("path"),
|
Field("path"),
|
||||||
Field("similarity", type="double"),
|
Field("similarity", type="double"),
|
||||||
Field("similaritypartial", type="double"),
|
Field("similaritypartial", type="double")
|
||||||
Field('last_modified', 'datetime')
|
|
||||||
)
|
|
||||||
|
|
||||||
db.define_table(
|
|
||||||
"analitic_llm_factura",
|
|
||||||
Field("content"),
|
|
||||||
Field("trusted"),
|
|
||||||
Field("model"),
|
|
||||||
Field("time", type="double"),
|
|
||||||
Field("path"),
|
|
||||||
Field("similarity", type="double"),
|
|
||||||
Field("similaritypartial", type="double"),
|
|
||||||
Field('last_modified', 'datetime')
|
|
||||||
)
|
|
||||||
|
|
||||||
db.define_table(
|
|
||||||
"analitic_llm_generaciontexto",
|
|
||||||
Field("content"),
|
|
||||||
Field("trusted"),
|
|
||||||
Field("model"),
|
|
||||||
Field("time", type="double"),
|
|
||||||
Field("path"),
|
|
||||||
Field("similarity", type="double"),
|
|
||||||
Field("similaritypartial", type="double"),
|
|
||||||
Field('last_modified', 'datetime')
|
|
||||||
)
|
|
||||||
|
|
||||||
db.define_table(
|
|
||||||
"analitic_llm_rag",
|
|
||||||
Field("content"),
|
|
||||||
Field("trusted"),
|
|
||||||
Field("model"),
|
|
||||||
Field("time", type="double"),
|
|
||||||
Field("path"),
|
|
||||||
Field("similarity", type="double"),
|
|
||||||
Field("similaritypartial", type="double"),
|
|
||||||
Field('last_modified', 'datetime')
|
|
||||||
)
|
)
|
||||||
105
gui.py
105
gui.py
|
|
@ -1,105 +0,0 @@
|
||||||
from taipy.gui import Gui
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import codecs, os
|
|
||||||
from taipy.gui import Html
|
|
||||||
import pandas as pd
|
|
||||||
import requests
|
|
||||||
import statistics
|
|
||||||
from databases import db
|
|
||||||
pwd = os.getcwd()
|
|
||||||
|
|
||||||
HTML = os.path.join(pwd,"html", "index.html")
|
|
||||||
file_read = codecs.open(HTML, "r", "utf-8")
|
|
||||||
index = file_read.read()
|
|
||||||
html_page_index = Html(index)
|
|
||||||
def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"):
|
|
||||||
configPath=os.path.join(os.getcwd(),relPath)
|
|
||||||
with open(configPath, 'r', encoding='utf-8') as file:
|
|
||||||
config = json.load(file)[nameModel]
|
|
||||||
Output= config[dataOut]
|
|
||||||
return Output
|
|
||||||
mode_list=extractConfig(nameModel="SystemData",dataOut="mode_list")
|
|
||||||
def getmetricvoice(model):
|
|
||||||
rows = db(db.analitic_voice.model==model).select()
|
|
||||||
rows_list = rows.as_list()
|
|
||||||
data=pd.DataFrame(rows_list)
|
|
||||||
durationL=list()
|
|
||||||
for i in rows_list:
|
|
||||||
durationL.append(db(db.trusted.path == i["path"] ).select().last().duration)
|
|
||||||
duration=statistics.mean(durationL)
|
|
||||||
time=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['time'].values[0]
|
|
||||||
similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0]
|
|
||||||
similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0]
|
|
||||||
efectivetime=time/duration
|
|
||||||
return ({"model":model,"duration":duration,"time":time,"similarity":similarity,"similaritypartial":similaritypartial,"efectivetime":efectivetime})
|
|
||||||
|
|
||||||
def html_getmetricvoice():
|
|
||||||
models=list()
|
|
||||||
for row in db().select(db.analitic_voice.model, distinct=True):
|
|
||||||
models.append(row.model)
|
|
||||||
data={}
|
|
||||||
for model in models:
|
|
||||||
data[model]=getmetricvoice(model)
|
|
||||||
data=pd.DataFrame(data).T
|
|
||||||
datafiles={}
|
|
||||||
for row in db().select(db.analitic_voice.ALL):
|
|
||||||
datafiles[row.id]=row.as_dict()
|
|
||||||
datafiles=pd.DataFrame(datafiles).T
|
|
||||||
html="""
|
|
||||||
<taipy:table>{data_voice}</taipy:table>
|
|
||||||
<taipy:table filter=True>{data_files_voice}</taipy:table>
|
|
||||||
"""
|
|
||||||
|
|
||||||
return html,data,datafiles
|
|
||||||
html_page_getmetricsvoice,data_voice,data_files_voices=html_getmetricvoice()
|
|
||||||
|
|
||||||
mode="voice"
|
|
||||||
modetypedata="audio"
|
|
||||||
file="id2"
|
|
||||||
def changemenu(mode):
|
|
||||||
if mode_list[mode]=="audio":
|
|
||||||
pathori="example/audio"
|
|
||||||
if mode_list[mode]=="factura":
|
|
||||||
pathori="example/factura"
|
|
||||||
if mode_list[mode]=="texto":
|
|
||||||
pathori="example/texto"
|
|
||||||
seltypedata=mode_list[mode]
|
|
||||||
dir_list = os.listdir(pathori)
|
|
||||||
|
|
||||||
return pathori,seltypedata,dir_list
|
|
||||||
|
|
||||||
|
|
||||||
def trustedallhtml(mode):
|
|
||||||
pathori,seltypedata,dir_list=changemenu(mode)
|
|
||||||
|
|
||||||
|
|
||||||
textmode=""
|
|
||||||
for modeused in mode_list.keys():
|
|
||||||
textmode=textmode+"('%s','%s'),"%(modeused,modeused)
|
|
||||||
html="""<taipy:selector lov="{[%s]}" dropdown True on_change=changemenu>{sel}</taipy:selector>"""%(textmode)
|
|
||||||
Sal=""
|
|
||||||
for i in dir_list:
|
|
||||||
temp="""('%s', '%s'),"""%(str(pwd+"/"+pathori+"/"+i),str(i))
|
|
||||||
Sal=Sal+temp
|
|
||||||
html2="""<taipy:selector lov="{[%s]}" dropdown True >{sel2}</taipy:selector>"""%(Sal)
|
|
||||||
return html+html2
|
|
||||||
|
|
||||||
html_page_trustedall = Html(trustedallhtml(mode))
|
|
||||||
#print(sel,sel2,seltypedata)
|
|
||||||
HTML = os.path.join(pwd,"html", "index.html")
|
|
||||||
file_read = codecs.open(HTML, "r", "utf-8")
|
|
||||||
index = file_read.read()
|
|
||||||
html_page_index = Html(index)
|
|
||||||
|
|
||||||
data=pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
|
|
||||||
|
|
||||||
pages = {
|
|
||||||
"/": html_page_index ,
|
|
||||||
"getmetricsvoice": Html(html_page_getmetricsvoice),
|
|
||||||
"trustedall":html_page_trustedall
|
|
||||||
}
|
|
||||||
|
|
||||||
app = Gui(pages=pages)
|
|
||||||
if __name__=="__main__":
|
|
||||||
app.run(use_reloader=True,port=7882, change_delay=1600)#state.imageActive2,
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<title>CIDITEL AI Playground</title>
|
|
||||||
<link rel="stylesheet" tyle="text-decoration: none;" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
|
|
||||||
<link rel="stylesheet" tyle="text-decoration: none;" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@24,400,0,0" />
|
|
||||||
<link rel="shortcut icon" tyle="text-decoration: none;" href="statics/icons/favicon.svg" type="image/x-icon">
|
|
||||||
<link rel="stylesheet" tyle="text-decoration: none;" href="statics/css/style.css">
|
|
||||||
<link rel="stylesheet" tyle="text-decoration: none;" href="statics/css/media-queries.css">
|
|
||||||
<link rel="preconnect" tyle="text-decoration: none;" href="https://fonts.googleapis.com">
|
|
||||||
<link rel="preconnect" tyle="text-decoration: none;" href="https://fonts.gstatic.com" crossorigin>
|
|
||||||
<link tyle="text-decoration: none;" href="https://fonts.googleapis.com/css2?family=Kanit:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&display=swap" rel="stylesheet">
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
|
|
||||||
|
|
||||||
<taipy:table>{data}</taipy:table>
|
|
||||||
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
125
requirements.txt
125
requirements.txt
|
|
@ -1,125 +0,0 @@
|
||||||
aiohttp==3.9.5
|
|
||||||
aiosignal==1.3.1
|
|
||||||
aniso8601==9.0.1
|
|
||||||
annotated-types==0.7.0
|
|
||||||
anyio==4.4.0
|
|
||||||
apispec==6.4.0
|
|
||||||
apispec-webframeworks==1.0.0
|
|
||||||
arrow==1.3.0
|
|
||||||
attrs==23.2.0
|
|
||||||
audioread==3.0.1
|
|
||||||
Automat==22.10.0
|
|
||||||
bidict==0.23.1
|
|
||||||
binaryornot==0.4.4
|
|
||||||
blinker==1.8.2
|
|
||||||
boto3==1.34.34
|
|
||||||
botocore==1.34.150
|
|
||||||
certifi==2024.2.2
|
|
||||||
chardet==5.2.0
|
|
||||||
charset-normalizer==3.3.2
|
|
||||||
click==8.1.7
|
|
||||||
constantly==23.10.4
|
|
||||||
cookiecutter==2.5.0
|
|
||||||
datasets==2.19.1
|
|
||||||
deepdiff==6.7.1
|
|
||||||
dill==0.3.8
|
|
||||||
dnspython==2.6.1
|
|
||||||
email_validator==2.2.0
|
|
||||||
et-xmlfile==1.1.0
|
|
||||||
evaluate==0.4.2
|
|
||||||
fastapi==0.111.0
|
|
||||||
fastapi-cli==0.0.4
|
|
||||||
filelock==3.14.0
|
|
||||||
Flask==3.0.2
|
|
||||||
Flask-Cors==4.0.0
|
|
||||||
Flask-RESTful==0.3.10
|
|
||||||
Flask-SocketIO==5.3.6
|
|
||||||
frozenlist==1.4.1
|
|
||||||
fsspec==2024.3.1
|
|
||||||
fuzzywuzzy==0.18.0
|
|
||||||
gevent==23.9.1
|
|
||||||
gevent-websocket==0.10.1
|
|
||||||
gitignore_parser==0.1.11
|
|
||||||
greenlet==3.0.3
|
|
||||||
h11==0.14.0
|
|
||||||
httpcore==1.0.5
|
|
||||||
httptools==0.6.1
|
|
||||||
httpx==0.27.0
|
|
||||||
huggingface-hub==0.23.2
|
|
||||||
hyperlink==21.0.0
|
|
||||||
idna==3.7
|
|
||||||
incremental==24.7.2
|
|
||||||
itsdangerous==2.2.0
|
|
||||||
Jinja2==3.1.4
|
|
||||||
jmespath==1.0.1
|
|
||||||
kthread==0.2.3
|
|
||||||
Levenshtein==0.25.1
|
|
||||||
Markdown==3.5.2
|
|
||||||
markdown-it-py==3.0.0
|
|
||||||
MarkupSafe==2.1.5
|
|
||||||
marshmallow==3.20.2
|
|
||||||
mdurl==0.1.2
|
|
||||||
multidict==6.0.5
|
|
||||||
multiprocess==0.70.16
|
|
||||||
mutagen==1.47.0
|
|
||||||
networkx==3.2.1
|
|
||||||
numpy==1.26.4
|
|
||||||
openpyxl==3.1.2
|
|
||||||
ordered-set==4.1.0
|
|
||||||
orjson==3.10.6
|
|
||||||
packaging==24.0
|
|
||||||
pandas==2.2.0
|
|
||||||
passlib==1.7.4
|
|
||||||
pyarrow==15.0.0
|
|
||||||
pyarrow-hotfix==0.6
|
|
||||||
pydal==20240713.1
|
|
||||||
pydantic==2.8.2
|
|
||||||
pydantic_core==2.20.1
|
|
||||||
Pygments==2.18.0
|
|
||||||
pymongo==4.6.1
|
|
||||||
python-dateutil==2.9.0.post0
|
|
||||||
python-dotenv==1.0.1
|
|
||||||
python-engineio==4.9.1
|
|
||||||
python-Levenshtein==0.25.1
|
|
||||||
python-multipart==0.0.9
|
|
||||||
python-slugify==8.0.4
|
|
||||||
python-socketio==5.11.3
|
|
||||||
pytz==2023.3.post1
|
|
||||||
PyYAML==6.0.1
|
|
||||||
rapidfuzz==3.9.4
|
|
||||||
requests==2.32.3
|
|
||||||
rich==13.7.1
|
|
||||||
s3transfer==0.10.2
|
|
||||||
shellingham==1.5.4
|
|
||||||
simple-websocket==1.0.0
|
|
||||||
six==1.16.0
|
|
||||||
sniffio==1.3.1
|
|
||||||
SQLAlchemy==2.0.25
|
|
||||||
starlette==0.37.2
|
|
||||||
taipy==3.1.1
|
|
||||||
taipy-config==3.1.1
|
|
||||||
taipy-core==3.1.1
|
|
||||||
taipy-gui==3.1.4
|
|
||||||
taipy-rest==3.1.1
|
|
||||||
taipy-templates==3.1.1
|
|
||||||
text-unidecode==1.3
|
|
||||||
toml==0.10.2
|
|
||||||
tqdm==4.66.4
|
|
||||||
Twisted==23.10.0
|
|
||||||
typer==0.12.3
|
|
||||||
types-python-dateutil==2.9.0.20240316
|
|
||||||
typing_extensions==4.12.0
|
|
||||||
tzdata==2024.1
|
|
||||||
tzlocal==5.2
|
|
||||||
ujson==5.10.0
|
|
||||||
urllib3==2.2.1
|
|
||||||
uvicorn==0.30.1
|
|
||||||
uvloop==0.19.0
|
|
||||||
watchfiles==0.22.0
|
|
||||||
websockets==12.0
|
|
||||||
Werkzeug==3.0.3
|
|
||||||
wsproto==1.2.0
|
|
||||||
xxhash==3.4.1
|
|
||||||
yarl==1.9.4
|
|
||||||
zope.event==5.0
|
|
||||||
zope.interface==6.4.post2
|
|
||||||
Loading…
Reference in New Issue