mgil
/
LLm2Node
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changes in model with finetuning

This commit is contained in:
Mario Gil 2023-10-28 06:36:39 -05:00
parent d7429cda09
commit 739036e038
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
main.py
View File

@ -24,8 +24,8 @@ from nltk.corpus import stopwords
from typing import Optional from typing import Optional
#from cleantext import clean #from cleantext import clean
import re import re
model="embeddings/all-mpnet-base-v2" model="FTv1/all-mpnet-base-v2"
entrenamiento="V0.0" entrenamiento="V1.0"
@ -93,7 +93,7 @@ def remove_unwanted(document,stopOK=False,punctuationOK=False,xtrasOK=False, emo
return document.strip().lower() return document.strip().lower()
def loadmodelEmb(model_name = "embeddings/all-MiniLM-L6-v2",model_kwargs = {'device': 'cpu'}): def loadmodelEmb(model_name = "embeddings/all-MiniLM-L6-v2",model_kwargs = {'device': 'cpu'}):
st = SentenceTransformer(model_name) st = SentenceTransformer(model_name,device='cpu')
return st return st
@ -138,7 +138,7 @@ def FinderDbs(query,dbs,filtred=1.2):
Sal = dbt.similarity_search_with_score(query,4) Sal = dbt.similarity_search_with_score(query,4)
for output in Sal: for output in Sal:
if output[0].metadata["id"] in AllData.keys(): if output[0].metadata["id"] in AllData.keys():
AllData[output[0].metadata["id"]]["d"]=min([AllData[output[0].metadata["id"]]["d"]-0.1,output[1]-0.1]) AllData[output[0].metadata["id"]]["d"]=min([AllData[output[0].metadata["id"]]["d"],output[1]])
else: else:
AllData[output[0].metadata["id"]]={"d":output[1],"page_content":output[0].page_content} AllData[output[0].metadata["id"]]={"d":output[1],"page_content":output[0].page_content}
#for item in AllData.items(): #for item in AllData.items():
@ -177,7 +177,7 @@ def calculate_api(response: Response):
except: except:
filtred = -9.0 filtred = -9.0
AllData=FinderDbs(query,[db2],filtred) AllData=FinderDbs(query,[db2,db],filtred)
versionL="_".join([model,entrenamiento]) versionL="_".join([model,entrenamiento])
if AllData: if AllData:
AllData = list(AllData) AllData = list(AllData)