Bump OTB Billing to v1.4.0 with crypto reconcile health card

PROJECT_STATE.md

@@ -1,5 +1,36 @@
 # PROJECT_STATE - OTB Billing
 
+## v1.4.0 - 2026-05-18
+
+Current state:
+- OTB Billing is running on outsidethedb.
+- Service: otb_billing.service.
+- App port: 5050.
+- Project path: /home/def/otb_billing.
+- /health renders the standard health grid with:
+  - Status
+  - Database
+  - Uptime
+  - Load Average
+  - Memory
+  - Disk
+  - Operations Bal
+  - Treasury Bal
+  - Crypto Reconcile
+- Operations Bal and Treasury Bal show USDC, ETH, ETHO, EGAZ, and ETI balances.
+- Coin names in balance cards are clickable and open the relevant explorer address page:
+  - Arbiscan for Arbitrum USDC
+  - Etherscan for Ethereum ETH
+  - explorer.ethoprotocol.com for ETHO
+  - explorer.etica-stats.org for EGAZ/ETI
+- Crypto Reconcile card shows timer/service state plus payment queue stats.
+- Reconcile Now button on /health manually starts the existing crypto reconciliation worker.
+- Crypto reconcile worker remains timer-driven by otb-billing-crypto-reconcile.timer every 15 minutes.
+- otb-billing-crypto-reconcile.service is a one-shot service; inactive between timer runs is normal.
+- Current version: v1.4.0.
+
+# PROJECT_STATE - OTB Billing
+
 ## v1.3.0 - 2026-05-17
 
 Current state:

README.md

@@ -1,3 +1,22 @@
+# OTB Billing - v1.4.0
+
+Build date: 2026-05-18
+
+## v1.4.0 changes
+
+- Added clickable explorer links for payment asset names in the Operations Bal and Treasury Bal cards.
+- Added Crypto Reconcile health card to the main /health grid.
+- Crypto Reconcile card now shows:
+  - timer status
+  - service status
+  - pending crypto payments
+  - confirmed crypto payments today
+  - stale pending crypto payments
+- Added Reconcile Now button on /health to manually trigger the existing crypto reconciliation worker.
+- Fixed crypto payment queue stats to use the project DB helper instead of guessing DB config.
+- Kept the reconcile worker as a systemd timer-triggered one-shot service; inactive service state between runs is expected when the timer is active.
+- Bumped footer/app version from v1.3.0 to v1.4.0.
+
 # OTB Billing - v1.3.0
 
 Build date: 2026-05-17

VERSION

@@ -1 +1 @@
-v1.3.0
+v1.4.0

backend/health.py

@@ -5,7 +5,7 @@ import platform
 from datetime import datetime, timezone
 from zoneinfo import ZoneInfo
 
-from flask import render_template, jsonify
+from flask import jsonify, redirect, render_template, request
 
 APP_START_TS = time.time()
 
@@ -217,6 +217,32 @@ def _wallet_erc20_balance(rpc_urls, token_contract, wallet_address, decimals):
     return _wallet_balance_format(int(result or "0x0", 16), decimals)
 
 
+
+
+def _wallet_explorer_url(coin, chain, wallet_address):
+    wallet = str(wallet_address or "").strip()
+    if not wallet:
+        return None
+
+    coin = str(coin or "").upper()
+    chain = str(chain or "").lower()
+
+    # Native/token balances are shown by wallet address on each chain explorer.
+    if coin == "USDC" or chain == "arbitrum":
+        return f"https://arbiscan.io/address/{wallet}"
+
+    if coin == "ETH" or chain == "ethereum":
+        return f"https://etherscan.io/address/{wallet}"
+
+    if coin == "ETHO" or chain == "etho":
+        return f"https://explorer.ethoprotocol.com/address/{wallet}"
+
+    if coin in ("EGAZ", "ETI") or chain == "etica":
+        return f"https://explorer.etica-stats.org/address/{wallet}"
+
+    return None
+
+
 def _wallet_balances(wallet_address=None, label=None):
     import os
 
@@ -307,6 +333,7 @@ def _wallet_balances(wallet_address=None, label=None):
             "balance": "error",
             "ok": False,
             "error": None,
+            "explorer_url": _wallet_explorer_url(asset["coin"], asset["chain"], wallet),
         }
 
         try:
@@ -337,6 +364,172 @@ def _wallet_balances(wallet_address=None, label=None):
 
 
 
+
+
+def _systemctl_value(args, timeout=3):
+    import subprocess
+
+    try:
+        proc = subprocess.run(
+            ["systemctl"] + list(args),
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=timeout,
+            check=False,
+        )
+        return {
+            "ok": proc.returncode == 0,
+            "code": proc.returncode,
+            "stdout": (proc.stdout or "").strip(),
+            "stderr": (proc.stderr or "").strip(),
+        }
+    except Exception as exc:
+        return {
+            "ok": False,
+            "code": None,
+            "stdout": "",
+            "stderr": str(exc),
+        }
+
+
+def _try_payment_stats_from_db(app):
+    stats = {
+        "available": False,
+        "error": None,
+        "pending": 0,
+        "confirmed_today": 0,
+        "stale_pending": 0,
+    }
+
+    conn = None
+
+    try:
+        from db import get_db_connection
+
+        conn = get_db_connection()
+        cur = conn.cursor(dictionary=True)
+
+        cur.execute("SHOW COLUMNS FROM payments")
+        cols = {row["Field"] for row in cur.fetchall()}
+
+        crypto_filters = []
+
+        if "payment_currency" in cols:
+            crypto_filters.append("payment_currency IN ('USDC','ETH','ETHO','EGAZ','ETI')")
+
+        if "notes" in cols:
+            crypto_filters.append("notes LIKE '%portal_crypto_intent:%'")
+
+        crypto_clause = ""
+        if crypto_filters:
+            crypto_clause = " AND (" + " OR ".join(crypto_filters) + ")"
+
+        if "payment_status" not in cols:
+            stats["error"] = "payments.payment_status column not found"
+            return stats
+
+        cur.execute(f"""
+            SELECT COUNT(*) AS count_value
+            FROM payments
+            WHERE payment_status = 'pending'
+            {crypto_clause}
+        """)
+        stats["pending"] = int((cur.fetchone() or {}).get("count_value") or 0)
+
+        timestamp_candidates = [
+            c for c in ("received_at", "updated_at", "created_at")
+            if c in cols
+        ]
+
+        if timestamp_candidates:
+            timestamp_expr = "COALESCE(" + ", ".join(timestamp_candidates) + ")"
+            cur.execute(f"""
+                SELECT COUNT(*) AS count_value
+                FROM payments
+                WHERE payment_status = 'confirmed'
+                  AND DATE({timestamp_expr}) = UTC_DATE()
+                {crypto_clause}
+            """)
+            stats["confirmed_today"] = int((cur.fetchone() or {}).get("count_value") or 0)
+        else:
+            stats["confirmed_today"] = 0
+
+        stale_col = None
+        for candidate in ("created_at", "first_seen_at", "last_checked_at"):
+            if candidate in cols:
+                stale_col = candidate
+                break
+
+        if stale_col:
+            cur.execute(f"""
+                SELECT COUNT(*) AS count_value
+                FROM payments
+                WHERE payment_status = 'pending'
+                  AND {stale_col} < (UTC_TIMESTAMP() - INTERVAL 30 MINUTE)
+                {crypto_clause}
+            """)
+            stats["stale_pending"] = int((cur.fetchone() or {}).get("count_value") or 0)
+        else:
+            stats["stale_pending"] = 0
+
+        stats["available"] = True
+        stats["error"] = None
+        return stats
+
+    except Exception as exc:
+        stats["available"] = False
+        stats["error"] = str(exc)[:220]
+        return stats
+
+    finally:
+        try:
+            if conn:
+                conn.close()
+        except Exception:
+            pass
+
+
+def _crypto_reconcile_status(app):
+    service_name = "otb-billing-crypto-reconcile.service"
+    timer_name = "otb-billing-crypto-reconcile.timer"
+
+    timer_active = _systemctl_value(["is-active", timer_name])
+    timer_enabled = _systemctl_value(["is-enabled", timer_name])
+    service_active = _systemctl_value(["is-active", service_name])
+    service_enabled = _systemctl_value(["is-enabled", service_name])
+
+    timer_list = _systemctl_value([
+        "list-timers",
+        "--all",
+        "--no-pager",
+        "--plain",
+        timer_name,
+    ])
+
+    last_logs = _systemctl_value([
+        "show",
+        service_name,
+        "--property=ActiveEnterTimestamp",
+        "--property=InactiveEnterTimestamp",
+        "--property=ExecMainStatus",
+        "--property=Result",
+        "--no-pager",
+    ])
+
+    return {
+        "service_name": service_name,
+        "timer_name": timer_name,
+        "timer_active": timer_active["stdout"] or "unknown",
+        "timer_enabled": timer_enabled["stdout"] or "unknown",
+        "service_active": service_active["stdout"] or "unknown",
+        "service_enabled": service_enabled["stdout"] or "unknown",
+        "timer_line": timer_list["stdout"],
+        "service_details": last_logs["stdout"],
+        "payment_stats": _try_payment_stats_from_db(app),
+    }
+
+
 def register_health_routes(app):
     @app.route("/health", methods=["GET"])
     def health_page():
@@ -351,8 +544,39 @@ def register_health_routes(app):
         )
         # Backward compatible JSON key for anything already reading health.json.
         health["wallet_balances"] = health["operations_balances"]
+        health["crypto_reconcile"] = _crypto_reconcile_status(app)
         return render_template("health.html", health=health), http_code
 
+
+    @app.route("/health/reconcile-now", methods=["POST"])
+    def health_reconcile_now():
+        import subprocess
+        import sys
+        from pathlib import Path
+
+        base_dir = Path("/home/def/otb_billing")
+        worker = base_dir / "scripts" / "crypto_reconciliation_worker.py"
+        log_path = base_dir / "logs" / "crypto_reconciliation_worker.log"
+
+        if not worker.exists():
+            return redirect("/health?reconcile=missing-worker")
+
+        try:
+            log_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(log_path, "a", encoding="utf-8") as log_fh:
+                log_fh.write("\\n[manual health trigger] starting crypto reconciliation worker\\n")
+                subprocess.Popen(
+                    [sys.executable, str(worker)],
+                    cwd=str(base_dir),
+                    stdout=log_fh,
+                    stderr=subprocess.STDOUT,
+                    start_new_session=True,
+                )
+            return redirect("/health?reconcile=started")
+        except Exception as exc:
+            return redirect("/health?reconcile=failed")
+
+
     @app.route("/health.json", methods=["GET"])
     def health_json():
         health, http_code = _health_payload(app)
@@ -366,4 +590,5 @@ def register_health_routes(app):
         )
         # Backward compatible JSON key for anything already reading health.json.
         health["wallet_balances"] = health["operations_balances"]
+        health["crypto_reconcile"] = _crypto_reconcile_status(app)
         return jsonify(health), http_code

templates/health.html

@@ -48,6 +48,46 @@
             margin-bottom: 0.85rem;
         }
 
+        .health-card a {
+            color: inherit;
+            text-decoration: underline;
+            text-underline-offset: 2px;
+        }
+
+        .ok {
+            color: #69db7c;
+            font-weight: 700;
+        }
+
+        .warn {
+            color: #ffd43b;
+            font-weight: 700;
+        }
+
+        .bad {
+            color: #ff8787;
+            font-weight: 700;
+        }
+
+        .health-action-button {
+            margin-top: 0.65rem;
+            padding: 0.45rem 0.7rem;
+            border: 1px solid rgba(255,255,255,0.25);
+            border-radius: 0.45rem;
+            background: rgba(255,255,255,0.08);
+            color: inherit;
+            cursor: pointer;
+        }
+
+        .health-action-button:hover {
+            background: rgba(255,255,255,0.14);
+        }
+
+        .health-note {
+            margin-top: 0.55rem;
+            opacity: 0.9;
+        }
+
         .health-card p {
             margin: 0.45rem 0;
         }
@@ -138,7 +178,7 @@
             {% if health.operations_balances %}
                 {% for asset in health.operations_balances.assets %}
                     <p>
-                        <strong>{{ asset.coin }}:</strong>
+                        <strong>{% if asset.explorer_url %}<a href="{{ asset.explorer_url }}" target="_blank" rel="noopener">{{ asset.coin }}</a>{% else %}{{ asset.coin }}{% endif %}:</strong>
                         {% if asset.ok %}
                             {{ asset.balance }}
                         {% else %}
@@ -157,7 +197,7 @@
             {% if health.treasury_balances %}
                 {% for asset in health.treasury_balances.assets %}
                     <p>
-                        <strong>{{ asset.coin }}:</strong>
+                        <strong>{% if asset.explorer_url %}<a href="{{ asset.explorer_url }}" target="_blank" rel="noopener">{{ asset.coin }}</a>{% else %}{{ asset.coin }}{% endif %}:</strong>
                         {% if asset.ok %}
                             {{ asset.balance }}
                         {% else %}
@@ -169,8 +209,52 @@
             {% else %}
                 <p>Not available</p>
             {% endif %}
+</div>
+    
+        <div class="health-card">
+            <h3>Crypto Reconcile</h3>
+            {% if health.crypto_reconcile %}
+                <p><strong>Timer:</strong>
+                    {% if health.crypto_reconcile.timer_active == "active" %}
+                        <span class="ok">{{ health.crypto_reconcile.timer_active }}</span>
+                    {% else %}
+                        <span class="warn">{{ health.crypto_reconcile.timer_active }}</span>
+                    {% endif %}
+                </p>
+                <p><strong>Service:</strong> {{ health.crypto_reconcile.service_active }}</p>
+
+                {% if health.crypto_reconcile.payment_stats and health.crypto_reconcile.payment_stats.available %}
+                    <p><strong>Pending:</strong> {{ health.crypto_reconcile.payment_stats.pending }}</p>
+                    <p><strong>Confirmed Today:</strong> {{ health.crypto_reconcile.payment_stats.confirmed_today }}</p>
+                    <p><strong>Stale Pending:</strong>
+                        {% if health.crypto_reconcile.payment_stats.stale_pending and health.crypto_reconcile.payment_stats.stale_pending > 0 %}
+                            <span class="warn">{{ health.crypto_reconcile.payment_stats.stale_pending }}</span>
+                        {% else %}
+                            {{ health.crypto_reconcile.payment_stats.stale_pending }}
+                        {% endif %}
+                    </p>
+                {% elif health.crypto_reconcile.payment_stats %}
+                    <p><strong>Stats:</strong> <span class="warn">unavailable</span></p>
+                    <p class="monoish"><strong>Error:</strong> {{ health.crypto_reconcile.payment_stats.error }}</p>
+                {% endif %}
+
+                <form method="post" action="/health/reconcile-now">
+                    <button class="health-action-button" type="submit">Reconcile Now</button>
+                </form>
+
+                {% if request.args.get("reconcile") == "started" %}
+                    <p class="health-note ok">Manual reconcile started.</p>
+                {% elif request.args.get("reconcile") == "failed" %}
+                    <p class="health-note bad">Manual reconcile failed to start.</p>
+                {% elif request.args.get("reconcile") == "missing-worker" %}
+                    <p class="health-note bad">Worker script missing.</p>
+                {% endif %}
+            {% else %}
+                <p>Not available</p>
+            {% endif %}
         </div>
-    </div>
+
+</div>
 </div>
 
 {% include "footer.html" %}