mgil
/
EvalDataSetHugging
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Metrics ok

This commit is contained in:
Mario Gil 2024-08-20 09:11:52 -05:00
parent 3514733885
commit 7a9a9f7de1
3 changed files with 391 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

442
apis.py
View File

@ -168,7 +168,7 @@ def addPrompt(response:Response5):
status_code=404,
content={"content": "mode no found" }
)
if mode == "llm_compra":
if mode == "llm_compra" or mode == "llm_generaciontexto":
hash1 = str(hashlib.sha256(prompt.encode()).hexdigest())
# with open("example/texto/"+hash1, 'w') as f:
# json.dump(info, f)
@ -556,6 +556,230 @@ def EvalLLMGeneracionTextohtml():
#
@app.get("/EvalFact")
@app.post("/EvalFact")
def EvalFact(response:Response1):
path=response.path
task_prompt=response.task_prompt
option=response.model
TrustedOCR=response.TrustedOCR
Trusted=TrustedOCR
if task_prompt=="":
if Trusted=="":
row=db(db.trusted.path == path and db.trusted.mode == "OCR").select().first()
try:
Trusted=row.trusted
except:
pass
Sal=main.EvalFacturas(path,task_prompt,TrustedOCR,option)
Sal["path"]=path
if db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).count()==0:
db.analitic_ocr.insert(**Sal)
db.commit()
else:
db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],jsonok=Sal["jsonok"])
db.commit()
return Sal
@app.get("/evalocrfactura")
def EvalOCRFactura():
dir_list = os.listdir(pathFact)
Sal=""
t=1
for i in dir_list:
temp="""<option value="%s">Opción %s, %s</option>
"""%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
Sal=Sal+temp
t=t+1
html="""<!DOCTYPE html>
<html lang="es">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Evaluacion de modelos OCR</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 20px;
}
input, button {
margin: 10px 0;
padding: 5px;
}
#respuesta {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
background-color: #f9f9f9;
}
</style>
</head>
<body>
<h1>Petición POST a API</h1>
<select id="texto1">
%s
</select>
<br>
<select id="texto2">
<option value="More Detailed Caption">More Detailed Caption</option>
<option value="OCR">OCR</option>
<option value="parsed">parsed</option>
<option value="scan">scan</option>
</select>
<br>
<input type="text" id="texto3" placeholder="TrustedOCR">
<br>
<input type="text" id="texto4" placeholder="option">
<br>
<button onclick="enviarPeticion()">Enviar petición</button>
<div id="respuesta"></div>
<script>
function enviarPeticion() {
const texto1 = document.getElementById('texto1').value;
const texto2 = document.getElementById('texto2').value;
const texto3 = document.getElementById('texto3').value;
const texto4 = document.getElementById('texto4').value;
const datos = {
path: texto1,
task_prompt: texto2,
TrustedOCR: texto3,
option: texto4
};
fetch('/EvalFact', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(datos)
})
.then(response => response.json())
.then(data => {
document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
})
.catch(error => {
document.getElementById('respuesta').innerHTML = 'Error: ' + error;
});
}
</script>
</body>
</html>
"""%(Sal)
return HTMLResponse(content=html, status_code=200)
@app.get("/evalllmfacturas")
def EvalllmFacturas():
dir_list = os.listdir(pathFact)
Sal=""
t=1
for i in dir_list:
temp="""<option value="%s">Opción %s, %s</option>
"""%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
Sal=Sal+temp
t=t+1
html="""<!DOCTYPE html>
<html lang="es">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Evaluacion modelos LLM</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 20px;
}
input, button {
margin: 10px 0;
padding: 5px;
}
#respuesta {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
background-color: #f9f9f9;
}
</style>
</head>
<body>
<h1>Petición POST a API</h1>
<select id="texto1">
%s
</select>
<br>
<select id="texto2">
<option value="">N.A.</option>
<option value="More Detailed Caption">More Detailed Caption</option>
<option value="OCR">OCR</option>
<option value="parsed">parsed</option>
<option value="scan">scan</option>
</select>
<br>
<input type="text" id="texto3" placeholder="system" value="Eres un chatbot amable">
<br>
<input type="text" id="texto4" placeholder="content" value="%s">
<br>
<input type="number" id="texto5" placeholder="max_tokens" value=1024>
<br>
<input type="text" id="texto6" placeholder="model" value="Claude-sonnet">
<br>
<input type="text" id="texto7" placeholder="prompt" value="Analiza la factura">
<br>
<input type="text" id="texto8" placeholder="TrustedLLmjson" value="{'A':''}">
<br>
<button onclick="enviarPeticion()">Enviar petición</button>
<div id="respuesta"></div>
<script>
function enviarPeticion() {
const texto1 = document.getElementById('texto1').value;
const texto2 = document.getElementById('texto2').value;
const texto3 = document.getElementById('texto3').value;
const texto4 = document.getElementById('texto4').value;
const texto5 = document.getElementById('texto5').value;
const texto6 = document.getElementById('texto6').value;
const texto7 = document.getElementById('texto7').value;
const texto8 = document.getElementById('texto8').value;
const datos = {
path: texto1,
task_prompt: texto2,
system: texto3,
content:texto4,
max_tokens:texto5,
model:texto6,
prompt:texto7,
TrustedLLmjson:texto8,
};
fetch('/EvalLLMFact', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(datos)
})
.then(response => response.json())
.then(data => {
document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
})
.catch(error => {
document.getElementById('respuesta').innerHTML = 'Error: ' + error;
});
}
</script>
</body>
</html>
"""%(Sal,"%s")
return HTMLResponse(content=html, status_code=200)
@ -707,33 +931,8 @@ display:flex;
@app.get("/EvalFact")
@app.post("/EvalFact")
def EvalFact(response:Response1):
path=response.path
task_prompt=response.task_prompt
option=response.model
TrustedOCR=response.TrustedOCR
Trusted=TrustedOCR
if task_prompt=="":
if Trusted=="":
row=db(db.trusted.path == path and db.trusted.mode == "OCR").select().first()
try:
Trusted=row.trusted
except:
pass
Sal=main.EvalFacturas(path,task_prompt,TrustedOCR,option)
Sal["path"]=path
if db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).count()==0:
db.analitic_ocr.insert(**Sal)
db.commit()
else:
db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],jsonok=Sal["jsonok"])
db.commit()
return Sal
@app.get("/EvalLLMFact")
@app.post("/EvalLLMFact")
def EvalLLMFact(response:Response2):
@ -751,93 +950,7 @@ def EvalLLMFact(response:Response2):
@app.get("/evalocrfactura")
def EvalOCRFactura():
dir_list = os.listdir(pathFact)
Sal=""
t=1
for i in dir_list:
temp="""<option value="%s">Opción %s, %s</option>
"""%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
Sal=Sal+temp
t=t+1
html="""<!DOCTYPE html>
<html lang="es">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Evaluacion de modelos OCR</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 20px;
}
input, button {
margin: 10px 0;
padding: 5px;
}
#respuesta {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
background-color: #f9f9f9;
}
</style>
</head>
<body>
<h1>Petición POST a API</h1>
<select id="texto1">
%s
</select>
<br>
<select id="texto2">
<option value="More Detailed Caption">More Detailed Caption</option>
<option value="OCR">OCR</option>
<option value="parsed">parsed</option>
<option value="scan">scan</option>
</select>
<br>
<input type="text" id="texto3" placeholder="TrustedOCR">
<br>
<input type="text" id="texto4" placeholder="option">
<br>
<button onclick="enviarPeticion()">Enviar petición</button>
<div id="respuesta"></div>
<script>
function enviarPeticion() {
const texto1 = document.getElementById('texto1').value;
const texto2 = document.getElementById('texto2').value;
const texto3 = document.getElementById('texto3').value;
const texto4 = document.getElementById('texto4').value;
const datos = {
path: texto1,
task_prompt: texto2,
TrustedOCR: texto3,
option: texto4
};
fetch('/EvalFact', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(datos)
})
.then(response => response.json())
.then(data => {
document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
})
.catch(error => {
document.getElementById('respuesta').innerHTML = 'Error: ' + error;
});
}
</script>
</body>
</html>
"""%(Sal)
return HTMLResponse(content=html, status_code=200)
def list2tablehtmlOCR(listdata,model):
html="""<h2>Table of {0}</h2>
@ -973,108 +1086,3 @@ display:flex;
@app.get("/evalllmfacturas")
def EvalllmFacturas():
dir_list = os.listdir(pathFact)
Sal=""
t=1
for i in dir_list:
temp="""<option value="%s">Opción %s, %s</option>
"""%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
Sal=Sal+temp
t=t+1
html="""<!DOCTYPE html>
<html lang="es">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Evaluacion modelos LLM</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 20px;
}
input, button {
margin: 10px 0;
padding: 5px;
}
#respuesta {
margin-top: 20px;
padding: 10px;
border: 1px solid #ccc;
background-color: #f9f9f9;
}
</style>
</head>
<body>
<h1>Petición POST a API</h1>
<select id="texto1">
%s
</select>
<br>
<select id="texto2">
<option value="">N.A.</option>
<option value="More Detailed Caption">More Detailed Caption</option>
<option value="OCR">OCR</option>
<option value="parsed">parsed</option>
<option value="scan">scan</option>
</select>
<br>
<input type="text" id="texto3" placeholder="system" value="Eres un chatbot amable">
<br>
<input type="text" id="texto4" placeholder="content" value="%s">
<br>
<input type="number" id="texto5" placeholder="max_tokens" value=1024>
<br>
<input type="text" id="texto6" placeholder="model" value="Claude-sonnet">
<br>
<input type="text" id="texto7" placeholder="prompt" value="Analiza la factura">
<br>
<input type="text" id="texto8" placeholder="TrustedLLmjson" value="{'A':''}">
<br>
<button onclick="enviarPeticion()">Enviar petición</button>
<div id="respuesta"></div>
<script>
function enviarPeticion() {
const texto1 = document.getElementById('texto1').value;
const texto2 = document.getElementById('texto2').value;
const texto3 = document.getElementById('texto3').value;
const texto4 = document.getElementById('texto4').value;
const texto5 = document.getElementById('texto5').value;
const texto6 = document.getElementById('texto6').value;
const texto7 = document.getElementById('texto7').value;
const texto8 = document.getElementById('texto8').value;
const datos = {
path: texto1,
task_prompt: texto2,
system: texto3,
content:texto4,
max_tokens:texto5,
model:texto6,
prompt:texto7,
TrustedLLmjson:texto8,
};
fetch('/EvalLLMFact', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(datos)
})
.then(response => response.json())
.then(data => {
document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
})
.catch(error => {
document.getElementById('respuesta').innerHTML = 'Error: ' + error;
});
}
</script>
</body>
</html>
"""%(Sal,"%s")
return HTMLResponse(content=html, status_code=200)

117
gui.py
View File

@ -120,9 +120,122 @@ def html_getmetricllm_compra():
return html,data,data_files
def getmetricllm_generaciontexto(model):
rows = db(db.analitic_llm_generaciontexto.model==model).select()
rows_list = rows.as_list()
data=pd.DataFrame(rows_list)
#durationL=list()
#for i in rows_list:
#durationL.append(db(db.trusted.path == i["path"] ).select().last().duration)
#duration=statistics.mean(durationL)
time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0]
relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0]
bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0]
toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0]
correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0]
#similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0]
#similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0]
#efectivetime=time/duration
return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness})
def html_getmetricllm_generaciontexto():
models=list()
t=time.time()
for row in db().select(db.analitic_llm_generaciontexto.model, distinct=True):
models.append(row.model)
data={}
for model in models:
data[model]=getmetricllm_generaciontexto(model)
data=pd.DataFrame(data).T
data_files={}
for row in db().select(db.analitic_llm_generaciontexto.ALL):
data_files[row.id]=row.as_dict()
data_files=pd.DataFrame(data_files).T
#table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'],
#columns=['model'], aggfunc="sum")
html="""
<h1>Data general de los modelos</h1>
<taipy:table>{data_llm_generaciontexto}</taipy:table>
<h1>Data de cada muestra</h1>
<taipy:table filter=True >{data_files_llm_generaciontexto}</taipy:table>
"""
#<taipy:chart mode="markers" x="x" y[1]="time" y[2]="similarity">{data_files_voice}</taipy:chart>
return html,data,data_files
def getmetricllm_factura(model):
rows = db(db.analitic_llm_factura.model==model).select()
rows_list = rows.as_list()
data=pd.DataFrame(rows_list)
#durationL=list()
#for i in rows_list:
#durationL.append(db(db.trusted.path == i["path"] ).select().last().duration)
#duration=statistics.mean(durationL)
time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0]
relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0]
bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0]
toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0]
correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0]
#similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0]
#similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0]
#efectivetime=time/duration
return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness})
def html_getmetricllm_factura():
models=list()
t=time.time()
for row in db().select(db.analitic_llm_factura.model, distinct=True):
models.append(row.model)
data={}
for model in models:
data[model]=getmetricllm_factura(model)
data=pd.DataFrame(data).T
data_files={}
for row in db().select(db.analitic_llm_factura.ALL):
data_files[row.id]=row.as_dict()
data_files=pd.DataFrame(data_files).T
#table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'],
#columns=['model'], aggfunc="sum")
html="""
<h1>Data general de los modelos</h1>
<taipy:table>{data_llm_factura}</taipy:table>
<h1>Data de cada muestra</h1>
<taipy:table filter=True >{data_files_llm_factura}</taipy:table>
"""
#<taipy:chart mode="markers" x="x" y[1]="time" y[2]="similarity">{data_files_voice}</taipy:chart>
return html,data,data_files
def on_init(state):
state.html_page_getmetricsvoice,state.data_voice,state.data_files_voice=html_getmetricvoice()
state.html_page_getmetricsllm_compra,state.data_llm_compra,state.data_files_llm_compra=html_getmetricllm_compra()
state.html_page_getmetricsllm_generaciontexto,state.data_llm_generaciontexto,state.data_files_llm_generaciontexto=html_getmetricllm_generaciontexto()
state.html_page_getmetricsllm_factura,state.data_llm_factura,state.data_files_llm_factura=html_getmetricllm_factura()
pass
@ -132,7 +245,9 @@ html_page_getmetricsvoice,data_voice,data_files_voice=html_getmetricvoice()
html_page_getmetricsllm_compra,data_llm_compra,data_files_llm_compra=html_getmetricllm_compra()
html_page_getmetricsllm_generaciontexto,data_llm_generaciontexto,data_files_llm_generaciontexto=html_getmetricllm_generaciontexto()
html_page_getmetricsllm_factura,data_llm_factura,data_files_llm_factura=html_getmetricllm_factura()
# mode="voice"
# modetypedata="audio"
# file="id2"
@ -156,6 +271,8 @@ html_page_getmetricsllm_compra,data_llm_compra,data_files_llm_compra=html_getmet
pages = {
"getmetricsvoice": Html(html_page_getmetricsvoice),
"getmetricsllm_compra": Html(html_page_getmetricsllm_compra),
"getmetricsllm_generaciontexto": Html(html_page_getmetricsllm_generaciontexto),
"getmetricsllm_factura": Html(html_page_getmetricsllm_factura)
}
app = Gui(pages=pages)

49
requirements.txt
View File

@ -2,9 +2,11 @@ aiohttp==3.9.5
aiosignal==1.3.1
aniso8601==9.0.1
annotated-types==0.7.0
anthropic==0.32.0
anyio==4.4.0
apispec==6.4.0
apispec-webframeworks==1.0.0
appdirs==1.4.4
arrow==1.3.0
attrs==23.2.0
audioread==3.0.1
@ -20,13 +22,20 @@ charset-normalizer==3.3.2
click==8.1.7
constantly==23.10.4
cookiecutter==2.5.0
dataclasses-json==0.6.7
datasets==2.19.1
deepdiff==6.7.1
deepeval==0.21.74
Deprecated==1.2.14
dill==0.3.8
distro==1.9.0
dnspython==2.6.1
docstring_parser==0.16
docx2txt==0.8
email_validator==2.2.0
et-xmlfile==1.1.0
evaluate==0.4.2
execnet==2.1.1
fastapi==0.111.0
fastapi-cli==0.0.4
filelock==3.14.0
@ -40,7 +49,9 @@ fuzzywuzzy==0.18.0
gevent==23.9.1
gevent-websocket==0.10.1
gitignore_parser==0.1.11
googleapis-common-protos==1.63.2
greenlet==3.0.3
grpcio==1.63.0
h11==0.14.0
httpcore==1.0.5
httptools==0.6.1
@ -48,11 +59,23 @@ httpx==0.27.0
huggingface-hub==0.23.2
hyperlink==21.0.0
idna==3.7
importlib-metadata==7.0.0
incremental==24.7.2
iniconfig==2.0.0
instructor==1.3.7
itsdangerous==2.2.0
Jinja2==3.1.4
jiter==0.4.2
jmespath==1.0.1
jsonpatch==1.33
jsonpointer==3.0.0
kthread==0.2.3
langchain==0.2.12
langchain-community==0.2.11
langchain-core==0.2.28
langchain-openai==0.1.20
langchain-text-splitters==0.2.2
langsmith==0.1.98
Levenshtein==0.25.1
Markdown==3.5.2
markdown-it-py==3.0.0
@ -62,14 +85,26 @@ mdurl==0.1.2
multidict==6.0.5
multiprocess==0.70.16
mutagen==1.47.0
mypy-extensions==1.0.0
nest-asyncio==1.6.0
networkx==3.2.1
numpy==1.26.4
openai==1.39.0
openpyxl==3.1.2
opentelemetry-api==1.24.0
opentelemetry-exporter-otlp-proto-common==1.24.0
opentelemetry-exporter-otlp-proto-grpc==1.24.0
opentelemetry-proto==1.24.0
opentelemetry-sdk==1.24.0
opentelemetry-semantic-conventions==0.45b0
ordered-set==4.1.0
orjson==3.10.6
packaging==24.0
pandas==2.2.0
passlib==1.7.4
pluggy==1.5.0
portalocker==2.10.1
protobuf==4.25.1
pyarrow==15.0.0
pyarrow-hotfix==0.6
pydal==20240713.1
@ -77,6 +112,10 @@ pydantic==2.8.2
pydantic_core==2.20.1
Pygments==2.18.0
pymongo==4.6.1
pysbd==0.3.4
pytest==8.3.2
pytest-repeat==0.9.3
pytest-xdist==3.6.1
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
python-engineio==4.9.1
@ -86,28 +125,36 @@ python-slugify==8.0.4
python-socketio==5.11.3
pytz==2023.3.post1
PyYAML==6.0.1
ragas==0.1.13
rapidfuzz==3.9.4
regex==2024.7.24
requests==2.32.3
rich==13.7.1
s3transfer==0.10.2
sentry-sdk==2.12.0
shellingham==1.5.4
simple-websocket==1.0.0
six==1.16.0
sniffio==1.3.1
SQLAlchemy==2.0.25
starlette==0.37.2
tabulate==0.9.0
taipy==3.1.1
taipy-config==3.1.1
taipy-core==3.1.1
taipy-gui==3.1.4
taipy-rest==3.1.1
taipy-templates==3.1.1
tenacity==8.4.2
text-unidecode==1.3
tiktoken==0.7.0
tokenizers==0.19.1
toml==0.10.2
tqdm==4.66.4
Twisted==23.10.0
typer==0.12.3
types-python-dateutil==2.9.0.20240316
typing-inspect==0.9.0
typing_extensions==4.12.0
tzdata==2024.1
tzlocal==5.2
@ -118,8 +165,10 @@ uvloop==0.19.0
watchfiles==0.22.0
websockets==12.0
Werkzeug==3.0.3
wrapt==1.16.0
wsproto==1.2.0
xxhash==3.4.1
yarl==1.9.4
zipp==3.19.2
zope.event==5.0
zope.interface==6.4.post2