Programming

1. Build Marts

"""Rebuild the panel, run the quality gate, rebuild every mart, sync to the DB."""
import time

from django.core.management.base import BaseCommand, CommandError

from lending_analytics import enrich as E
from lending_analytics import marts as M
from lending_analytics import quality as Q
from portfolio import bridge


class Command(BaseCommand):
    help = "Rebuild marts from the lake and load them into the database."

    def add_arguments(self, parser):
        parser.add_argument("--fx", default="nominal", choices=["nominal", "constant"])
        parser.add_argument("--skip-gate", action="store_true",
                            help="build even if a FAIL check fires (use knowingly)")
        parser.add_argument("--no-sync", action="store_true",
                            help="write parquet marts only, do not touch the DB")

    def handle(self, *args, **o):
        t0 = time.time()

        panel = E.build_panel(fx_mode=o["fx"])
        self.stdout.write(f"panel: {len(panel):,} loan-months, "
                          f"{panel['anketa'].nunique():,} loans "
                          f"[{time.time() - t0:.1f}s]")

        checks = Q.run_checks(panel)
        for r in checks[checks["status"] != "pass"].itertuples():
            style = self.style.ERROR if r.status == "FAIL" else self.style.WARNING
            self.stdout.write(style(f"  [{r.status}] {r.check}: {r.detail}"))
        if not o["skip_gate"]:
            try:
                Q.assert_gate(checks)
            except RuntimeError as exc:
                raise CommandError(str(exc))

        M.build_all(panel)
        self.stdout.write(f"marts built [{time.time() - t0:.1f}s]")

        if not o["no_sync"]:
            bridge.register_snapshots(panel)
            counts = bridge.sync_all(verbose=self.stdout.write)
            self.stdout.write(self.style.SUCCESS(
                f"synced {sum(counts.values()):,} rows into the database "
                f"[{time.time() - t0:.1f}s]"))
Helpful? Dislike 0 Log in to react