65 lines
1.9 KiB
Python
65 lines
1.9 KiB
Python
import os
|
|
from sqlalchemy import create_engine, Column, String, Float, MetaData, Table
|
|
from sqlalchemy.ext.declarative import declarative_base
|
|
from sqlalchemy.orm import sessionmaker
|
|
from article_analyzer import ArticleAnalyzer
|
|
|
|
Base = declarative_base()
|
|
|
|
class ArticleAnalysis(Base):
|
|
__tablename__ = 'article_analyses'
|
|
|
|
filename = Column(String, primary_key=True)
|
|
label = Column(String)
|
|
score = Column(Float)
|
|
|
|
def read_html_files(folder_path):
|
|
html_contents = {}
|
|
for root, _, files in os.walk(folder_path):
|
|
for file in files:
|
|
if file.endswith(".html"):
|
|
file_path = os.path.join(root, file)
|
|
with open(file_path, "r", encoding="utf-8") as f:
|
|
html_contents[file_path] = f.read()
|
|
return html_contents
|
|
|
|
|
|
if __name__ == "__main__":
|
|
analyzer = ArticleAnalyzer()
|
|
|
|
engine = create_engine('sqlite:///article_analysis.db')
|
|
Session = sessionmaker(bind=engine)
|
|
session = Session()
|
|
|
|
html_files = read_html_files("./data")
|
|
print(f"Parsed {len(html_files)} html files")
|
|
|
|
Base.metadata.create_all(engine)
|
|
|
|
|
|
for file, content in html_files.items():
|
|
result = analyzer.classify_article_finbert(content)
|
|
|
|
filename = os.path.basename(file)
|
|
|
|
label = result[0]['label']
|
|
score = result[0]['score']
|
|
|
|
analysis = ArticleAnalysis(filename=filename, label=label, score=score)
|
|
|
|
try:
|
|
session.add(analysis)
|
|
session.commit()
|
|
except:
|
|
session.rollback()
|
|
|
|
existing = session.query(ArticleAnalysis).filter_by(filename=filename).first()
|
|
if existing:
|
|
existing.label = label
|
|
existing.score = score
|
|
session.commit()
|
|
finally:
|
|
session.close()
|
|
|
|
print(f"article [{file}] - analyzed as [{result}]\n")
|