Simon/CryptoMarketParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chunked content to 512

Browse Source
This commit is contained in:
Simon Moisy 2025-03-22 08:00:58 +08:00
parent 3864d7e93c
commit 679f1bd941
1 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
main_article_analyzer.py
View File

@ -44,33 +44,41 @@ if __name__ == "__main__":
Base.metadata.create_all(engine)
result = analyzer.classify_article_finbert("Strong earning growth and expending market shares have positionned the company for long term success.")
print(f'result {result}')
# result = analyzer.classify_article_finbert("Strong earning growth and expending market shares have positionned the company for long term success.")
# print(f'result {result}')
for file, content in html_files.items():
chunk_size = 512
chunks = [content[i:i+chunk_size] for i in range(0, len(content), chunk_size)]
# for file, content in html_files.items():
# result = analyzer.classify_article_finbert(content)
results = []
for chunk in chunks:
if chunk.strip():
chunk_result = analyzer.classify_article_finbert(chunk)
results.extend(chunk_result)
# filename = os.path.basename(file)
# print(f'result {result}')
result = results if results else [{'label': 'neutral', 'score': 0.0}]
# label = result[0]['label']
# score = result[0]['score']
filename = os.path.basename(file)
print(f'result {result}')
# analysis = ArticleAnalysis(filename=filename, label=label, score=score)
# label = result[0]['label']
# score = result[0]['score']
# try:
# session.add(analysis)
# session.commit()
# except:
# session.rollback()
# analysis = ArticleAnalysis(filename=filename, label=label, score=score)
# existing = session.query(ArticleAnalysis).filter_by(filename=filename).first()
# if existing:
# existing.label = label
# existing.score = score
# session.commit()
# finally:
# session.close()
# try:
# session.add(analysis)
# session.commit()
# except:
# session.rollback()
# print(f"article [{file}] - analyzed as [{result}]\n")
# existing = session.query(ArticleAnalysis).filter_by(filename=filename).first()
# if existing:
# existing.label = label
# existing.score = score
# session.commit()
# finally:
# session.close()
# print(f"article [{file}] - analyzed as [{result}]\n")