CryptoMarketParser/main_article_analyzer.py

import os
import sys
from sqlalchemy import create_engine, Column, String, Float, MetaData, Table
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from article_analyzer import ArticleAnalyzer

sys.path.append(os.path.join(os.path.dirname(__file__), 'finBERT', 'finbert'))

Base = declarative_base()

class ArticleAnalysis(Base):
    __tablename__ = 'article_analyses'
    
    filename = Column(String, primary_key=True)
    label = Column(String)
    score = Column(Float)
    
def read_html_files(folder_path):
    html_contents = {}
    for root, _, files in os.walk(folder_path):
        for file in files:
            if file.endswith(".html"):
                file_path = os.path.join(root, file)
                with open(file_path, "r", encoding="utf-8") as f:
                    html_contents[file_path] = f.read()
    return html_contents


if __name__ == "__main__":
    analyzer = ArticleAnalyzer()

    engine = create_engine('sqlite:///article_analysis.db')
    Session = sessionmaker(bind=engine)
    session = Session()

    html_files = read_html_files("./data")
    print(f"Parsed {len(html_files)} html files")
    
    Base.metadata.create_all(engine)

    
    for file, content in html_files.items():
        result = analyzer.classify_article_finbert(content)

        filename = os.path.basename(file)
        
        label = result[0]['label']
        score = result[0]['score']

        analysis = ArticleAnalysis(filename=filename, label=label, score=score)
        
        try:
            session.add(analysis)
            session.commit()
        except:
            session.rollback()

            existing = session.query(ArticleAnalysis).filter_by(filename=filename).first()
            if existing:
                existing.label = label
                existing.score = score
                session.commit()
        finally:
            session.close()

        print(f"article [{file}] - analyzed as [{result}]\n")