Simon/CryptoMarketParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test calls to finbert working - need to chunk at 512

Browse Source
This commit is contained in:
Jericho 2025-03-22 06:25:14 +08:00
parent b4ef1ad8a2
commit 3864d7e93c
1 changed files with 33 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
main_article_analyzer.py
View File

@ -4,8 +4,8 @@ from sqlalchemy import create_engine, Column, String, Float, MetaData, Table
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from article_analyzer import ArticleAnalyzer from article_analyzer import ArticleAnalyzer
import nltk
sys.path.append(os.path.join(os.path.dirname(__file__), 'finBERT', 'finbert')) import logging
Base = declarative_base() Base = declarative_base()
@ -28,6 +28,11 @@ def read_html_files(folder_path):
if __name__ == "__main__": if __name__ == "__main__":
# nltk.set_proxy('http://127.0.0.1:7890')
# nltk.download('punkt_tab')
logging.basicConfig(level=logging.CRITICAL)
analyzer = ArticleAnalyzer() analyzer = ArticleAnalyzer()
engine = create_engine('sqlite:///article_analysis.db') engine = create_engine('sqlite:///article_analysis.db')
@ -39,29 +44,33 @@ if __name__ == "__main__":
Base.metadata.create_all(engine) Base.metadata.create_all(engine)
result = analyzer.classify_article_finbert("Strong earning growth and expending market shares have positionned the company for long term success.")
for file, content in html_files.items(): print(f'result {result}')
result = analyzer.classify_article_finbert(content)
filename = os.path.basename(file)
# for file, content in html_files.items():
# result = analyzer.classify_article_finbert(content)
# filename = os.path.basename(file)
# print(f'result {result}')
# label = result[0]['label']
# score = result[0]['score']
# analysis = ArticleAnalysis(filename=filename, label=label, score=score)
label = result[0]['label'] # try:
score = result[0]['score'] # session.add(analysis)
# session.commit()
# except:
# session.rollback()
analysis = ArticleAnalysis(filename=filename, label=label, score=score) # existing = session.query(ArticleAnalysis).filter_by(filename=filename).first()
# if existing:
try: # existing.label = label
session.add(analysis) # existing.score = score
session.commit() # session.commit()
except: # finally:
session.rollback() # session.close()
existing = session.query(ArticleAnalysis).filter_by(filename=filename).first() # print(f"article [{file}] - analyzed as [{result}]\n")
if existing:
existing.label = label
existing.score = score
session.commit()
finally:
session.close()
print(f"article [{file}] - analyzed as [{result}]\n")