From 7a9a9f7de1334456064f24b6dcafe7b837e55c77 Mon Sep 17 00:00:00 2001 From: marioggil Date: Tue, 20 Aug 2024 09:11:52 -0500 Subject: [PATCH] feat: Metrics ok --- apis.py | 442 ++++++++++++++++++++++++----------------------- gui.py | 117 +++++++++++++ requirements.txt | 49 ++++++ 3 files changed, 391 insertions(+), 217 deletions(-) diff --git a/apis.py b/apis.py index 5b53bc4..c5928eb 100644 --- a/apis.py +++ b/apis.py @@ -168,7 +168,7 @@ def addPrompt(response:Response5): status_code=404, content={"content": "mode no found" } ) - if mode == "llm_compra": + if mode == "llm_compra" or mode == "llm_generaciontexto": hash1 = str(hashlib.sha256(prompt.encode()).hexdigest()) # with open("example/texto/"+hash1, 'w') as f: # json.dump(info, f) @@ -556,6 +556,230 @@ def EvalLLMGeneracionTextohtml(): +# + +@app.get("/EvalFact") +@app.post("/EvalFact") +def EvalFact(response:Response1): + path=response.path + task_prompt=response.task_prompt + option=response.model + TrustedOCR=response.TrustedOCR + Trusted=TrustedOCR + if task_prompt=="": + if Trusted=="": + row=db(db.trusted.path == path and db.trusted.mode == "OCR").select().first() + try: + Trusted=row.trusted + except: + pass + Sal=main.EvalFacturas(path,task_prompt,TrustedOCR,option) + Sal["path"]=path + if db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).count()==0: + db.analitic_ocr.insert(**Sal) + db.commit() + else: + db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],jsonok=Sal["jsonok"]) + db.commit() + return Sal + + +@app.get("/evalocrfactura") +def EvalOCRFactura(): + dir_list = os.listdir(pathFact) + Sal="" + t=1 + for i in dir_list: + temp=""" + """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i)) + Sal=Sal+temp + t=t+1 + html=""" + + + + + Evaluacion de modelos OCR + + + +

Petición POST a API

+ +
+ + +
+ +
+ +
+ +
+ + + + + """%(Sal) + return HTMLResponse(content=html, status_code=200) + + + +@app.get("/evalllmfacturas") +def EvalllmFacturas(): + dir_list = os.listdir(pathFact) + Sal="" + t=1 + for i in dir_list: + temp=""" + """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i)) + Sal=Sal+temp + t=t+1 + html=""" + + + + + Evaluacion modelos LLM + + + +

Petición POST a API

+ +
+ + +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ + + + + """%(Sal,"%s") + return HTMLResponse(content=html, status_code=200) + @@ -707,33 +931,8 @@ display:flex; -@app.get("/EvalFact") -@app.post("/EvalFact") -def EvalFact(response:Response1): - path=response.path - task_prompt=response.task_prompt - option=response.model - TrustedOCR=response.TrustedOCR - Trusted=TrustedOCR - if task_prompt=="": - if Trusted=="": - row=db(db.trusted.path == path and db.trusted.mode == "OCR").select().first() - try: - Trusted=row.trusted - except: - pass - Sal=main.EvalFacturas(path,task_prompt,TrustedOCR,option) - Sal["path"]=path - if db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).count()==0: - db.analitic_ocr.insert(**Sal) - db.commit() - else: - db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],jsonok=Sal["jsonok"]) - db.commit() - return Sal - @app.get("/EvalLLMFact") @app.post("/EvalLLMFact") def EvalLLMFact(response:Response2): @@ -751,93 +950,7 @@ def EvalLLMFact(response:Response2): -@app.get("/evalocrfactura") -def EvalOCRFactura(): - dir_list = os.listdir(pathFact) - Sal="" - t=1 - for i in dir_list: - temp=""" - """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i)) - Sal=Sal+temp - t=t+1 - html=""" - - - - - Evaluacion de modelos OCR - - - -

Petición POST a API

- -
- -
- -
- -
- -
- - - - - """%(Sal) - return HTMLResponse(content=html, status_code=200) def list2tablehtmlOCR(listdata,model): html="""

Table of {0}

@@ -973,108 +1086,3 @@ display:flex; -@app.get("/evalllmfacturas") -def EvalllmFacturas(): - dir_list = os.listdir(pathFact) - Sal="" - t=1 - for i in dir_list: - temp=""" - """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i)) - Sal=Sal+temp - t=t+1 - html=""" - - - - - Evaluacion modelos LLM - - - -

Petición POST a API

- -
- - -
- -
- -
- -
- -
- -
- -
- -
- - - - - """%(Sal,"%s") - return HTMLResponse(content=html, status_code=200) \ No newline at end of file diff --git a/gui.py b/gui.py index 01c9e86..c2e2d0b 100644 --- a/gui.py +++ b/gui.py @@ -120,9 +120,122 @@ def html_getmetricllm_compra(): return html,data,data_files +def getmetricllm_generaciontexto(model): + rows = db(db.analitic_llm_generaciontexto.model==model).select() + rows_list = rows.as_list() + data=pd.DataFrame(rows_list) + + #durationL=list() + #for i in rows_list: + #durationL.append(db(db.trusted.path == i["path"] ).select().last().duration) + #duration=statistics.mean(durationL) + time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0] + relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0] + bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0] + toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0] + + correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0] + #similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0] + #similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0] + #efectivetime=time/duration + return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness}) + +def html_getmetricllm_generaciontexto(): + models=list() + t=time.time() + for row in db().select(db.analitic_llm_generaciontexto.model, distinct=True): + models.append(row.model) + data={} + for model in models: + data[model]=getmetricllm_generaciontexto(model) + data=pd.DataFrame(data).T + data_files={} + for row in db().select(db.analitic_llm_generaciontexto.ALL): + data_files[row.id]=row.as_dict() + + data_files=pd.DataFrame(data_files).T + + #table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'], + #columns=['model'], aggfunc="sum") + + + html=""" +

Data general de los modelos

+ {data_llm_generaciontexto} +

Data de cada muestra

+ {data_files_llm_generaciontexto} + + + """ + #{data_files_voice} + + return html,data,data_files + + + + +def getmetricllm_factura(model): + rows = db(db.analitic_llm_factura.model==model).select() + rows_list = rows.as_list() + data=pd.DataFrame(rows_list) + + #durationL=list() + #for i in rows_list: + #durationL.append(db(db.trusted.path == i["path"] ).select().last().duration) + #duration=statistics.mean(durationL) + time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0] + relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0] + bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0] + toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0] + + correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0] + #similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0] + #similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0] + #efectivetime=time/duration + return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness}) + + + +def html_getmetricllm_factura(): + models=list() + t=time.time() + for row in db().select(db.analitic_llm_factura.model, distinct=True): + models.append(row.model) + data={} + for model in models: + data[model]=getmetricllm_factura(model) + data=pd.DataFrame(data).T + data_files={} + for row in db().select(db.analitic_llm_factura.ALL): + data_files[row.id]=row.as_dict() + + data_files=pd.DataFrame(data_files).T + + #table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'], + #columns=['model'], aggfunc="sum") + + + html=""" +

Data general de los modelos

+ {data_llm_factura} +

Data de cada muestra

+ {data_files_llm_factura} + + + """ + #{data_files_voice} + + return html,data,data_files + + + + + def on_init(state): state.html_page_getmetricsvoice,state.data_voice,state.data_files_voice=html_getmetricvoice() state.html_page_getmetricsllm_compra,state.data_llm_compra,state.data_files_llm_compra=html_getmetricllm_compra() + state.html_page_getmetricsllm_generaciontexto,state.data_llm_generaciontexto,state.data_files_llm_generaciontexto=html_getmetricllm_generaciontexto() + state.html_page_getmetricsllm_factura,state.data_llm_factura,state.data_files_llm_factura=html_getmetricllm_factura() pass @@ -132,7 +245,9 @@ html_page_getmetricsvoice,data_voice,data_files_voice=html_getmetricvoice() html_page_getmetricsllm_compra,data_llm_compra,data_files_llm_compra=html_getmetricllm_compra() +html_page_getmetricsllm_generaciontexto,data_llm_generaciontexto,data_files_llm_generaciontexto=html_getmetricllm_generaciontexto() +html_page_getmetricsllm_factura,data_llm_factura,data_files_llm_factura=html_getmetricllm_factura() # mode="voice" # modetypedata="audio" # file="id2" @@ -156,6 +271,8 @@ html_page_getmetricsllm_compra,data_llm_compra,data_files_llm_compra=html_getmet pages = { "getmetricsvoice": Html(html_page_getmetricsvoice), "getmetricsllm_compra": Html(html_page_getmetricsllm_compra), + "getmetricsllm_generaciontexto": Html(html_page_getmetricsllm_generaciontexto), + "getmetricsllm_factura": Html(html_page_getmetricsllm_factura) } app = Gui(pages=pages) diff --git a/requirements.txt b/requirements.txt index a41c6d9..215eee0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,11 @@ aiohttp==3.9.5 aiosignal==1.3.1 aniso8601==9.0.1 annotated-types==0.7.0 +anthropic==0.32.0 anyio==4.4.0 apispec==6.4.0 apispec-webframeworks==1.0.0 +appdirs==1.4.4 arrow==1.3.0 attrs==23.2.0 audioread==3.0.1 @@ -20,13 +22,20 @@ charset-normalizer==3.3.2 click==8.1.7 constantly==23.10.4 cookiecutter==2.5.0 +dataclasses-json==0.6.7 datasets==2.19.1 deepdiff==6.7.1 +deepeval==0.21.74 +Deprecated==1.2.14 dill==0.3.8 +distro==1.9.0 dnspython==2.6.1 +docstring_parser==0.16 +docx2txt==0.8 email_validator==2.2.0 et-xmlfile==1.1.0 evaluate==0.4.2 +execnet==2.1.1 fastapi==0.111.0 fastapi-cli==0.0.4 filelock==3.14.0 @@ -40,7 +49,9 @@ fuzzywuzzy==0.18.0 gevent==23.9.1 gevent-websocket==0.10.1 gitignore_parser==0.1.11 +googleapis-common-protos==1.63.2 greenlet==3.0.3 +grpcio==1.63.0 h11==0.14.0 httpcore==1.0.5 httptools==0.6.1 @@ -48,11 +59,23 @@ httpx==0.27.0 huggingface-hub==0.23.2 hyperlink==21.0.0 idna==3.7 +importlib-metadata==7.0.0 incremental==24.7.2 +iniconfig==2.0.0 +instructor==1.3.7 itsdangerous==2.2.0 Jinja2==3.1.4 +jiter==0.4.2 jmespath==1.0.1 +jsonpatch==1.33 +jsonpointer==3.0.0 kthread==0.2.3 +langchain==0.2.12 +langchain-community==0.2.11 +langchain-core==0.2.28 +langchain-openai==0.1.20 +langchain-text-splitters==0.2.2 +langsmith==0.1.98 Levenshtein==0.25.1 Markdown==3.5.2 markdown-it-py==3.0.0 @@ -62,14 +85,26 @@ mdurl==0.1.2 multidict==6.0.5 multiprocess==0.70.16 mutagen==1.47.0 +mypy-extensions==1.0.0 +nest-asyncio==1.6.0 networkx==3.2.1 numpy==1.26.4 +openai==1.39.0 openpyxl==3.1.2 +opentelemetry-api==1.24.0 +opentelemetry-exporter-otlp-proto-common==1.24.0 +opentelemetry-exporter-otlp-proto-grpc==1.24.0 +opentelemetry-proto==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 ordered-set==4.1.0 orjson==3.10.6 packaging==24.0 pandas==2.2.0 passlib==1.7.4 +pluggy==1.5.0 +portalocker==2.10.1 +protobuf==4.25.1 pyarrow==15.0.0 pyarrow-hotfix==0.6 pydal==20240713.1 @@ -77,6 +112,10 @@ pydantic==2.8.2 pydantic_core==2.20.1 Pygments==2.18.0 pymongo==4.6.1 +pysbd==0.3.4 +pytest==8.3.2 +pytest-repeat==0.9.3 +pytest-xdist==3.6.1 python-dateutil==2.9.0.post0 python-dotenv==1.0.1 python-engineio==4.9.1 @@ -86,28 +125,36 @@ python-slugify==8.0.4 python-socketio==5.11.3 pytz==2023.3.post1 PyYAML==6.0.1 +ragas==0.1.13 rapidfuzz==3.9.4 +regex==2024.7.24 requests==2.32.3 rich==13.7.1 s3transfer==0.10.2 +sentry-sdk==2.12.0 shellingham==1.5.4 simple-websocket==1.0.0 six==1.16.0 sniffio==1.3.1 SQLAlchemy==2.0.25 starlette==0.37.2 +tabulate==0.9.0 taipy==3.1.1 taipy-config==3.1.1 taipy-core==3.1.1 taipy-gui==3.1.4 taipy-rest==3.1.1 taipy-templates==3.1.1 +tenacity==8.4.2 text-unidecode==1.3 +tiktoken==0.7.0 +tokenizers==0.19.1 toml==0.10.2 tqdm==4.66.4 Twisted==23.10.0 typer==0.12.3 types-python-dateutil==2.9.0.20240316 +typing-inspect==0.9.0 typing_extensions==4.12.0 tzdata==2024.1 tzlocal==5.2 @@ -118,8 +165,10 @@ uvloop==0.19.0 watchfiles==0.22.0 websockets==12.0 Werkzeug==3.0.3 +wrapt==1.16.0 wsproto==1.2.0 xxhash==3.4.1 yarl==1.9.4 +zipp==3.19.2 zope.event==5.0 zope.interface==6.4.post2