#!/usr/bin/env python3
"""
SWAPA Email Analytics — Data Fetcher
Pulls campaign performance stats from the Constant Contact v3 API.
Outputs: data/email_stats.json  ← read by swapa_email_dashboard.html

Normal usage (runs via launchd every Monday at 9:30am):
  python3 fetch_email_stats.py
"""

import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from base64 import b64encode
from collections import defaultdict
from datetime import datetime, timedelta, timezone
from pathlib import Path

# ── CONFIG ────────────────────────────────────────────────────────────────────
CLIENT_ID     = "65b87674-4697-486a-8d58-b3e2e6f01d4f"
CLIENT_SECRET = "jv-07NGjA_91dprCebmKZw"
# Long-lived refresh token from get_email_oauth_token.py. Only needs replacing
# if revoked at Constant Contact's My Applications page or the secret is regenerated.
REFRESH_TOKEN = "eT15xRffEEqzrD9-ctlA-kOE70dVc5GBV9pcMFlPK-o"

TOKEN_URL = "https://authz.constantcontact.com/oauth2/default/v1/token"
API_ROOT  = "https://api.cc.email"
API_BASE  = f"{API_ROOT}/v3"

# Constant Contact's API sits behind Cloudflare, which blocks the default
# urllib User-Agent (see get_email_oauth_token.py) — always send a browser-like one.
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"

OUTPUT_DIR = Path(__file__).parent / "data"

# ── HTTP HELPERS ─────────────────────────────────────────────────────────────
def http_get_json(url, headers=None, retries=3):
    req = urllib.request.Request(url)
    req.add_header("User-Agent", USER_AGENT)
    req.add_header("Accept", "application/json")
    for k, v in (headers or {}).items():
        req.add_header(k, v)
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                return json.loads(r.read())
        except urllib.error.HTTPError as e:
            if e.code == 429 and attempt < retries - 1:
                time.sleep(2 * (attempt + 1))
                continue
            body = e.read().decode(errors="replace")
            raise RuntimeError(f"GET {url} failed: HTTP {e.code}\n{body}") from None

def http_post_form(url, data, headers=None):
    encoded = urllib.parse.urlencode(data).encode()
    req = urllib.request.Request(url, data=encoded)
    req.add_header("Content-Type", "application/x-www-form-urlencoded")
    req.add_header("User-Agent", USER_AGENT)
    req.add_header("Accept", "application/json")
    for k, v in (headers or {}).items():
        req.add_header(k, v)
    try:
        with urllib.request.urlopen(req, timeout=30) as r:
            return json.loads(r.read())
    except urllib.error.HTTPError as e:
        body = e.read().decode(errors="replace")
        raise RuntimeError(f"POST {url} failed: HTTP {e.code}\n{body}") from None

# ── AUTH ──────────────────────────────────────────────────────────────────────
def get_access_token():
    basic_auth = b64encode(f"{CLIENT_ID}:{CLIENT_SECRET}".encode()).decode()
    resp = http_post_form(TOKEN_URL, {
        "refresh_token": REFRESH_TOKEN,
        "grant_type":    "refresh_token",
    }, headers={"Authorization": f"Basic {basic_auth}"})
    token = resp.get("access_token")
    if not token:
        raise RuntimeError(f"Constant Contact auth failed: {resp}")
    return token

# ── CAMPAIGNS ─────────────────────────────────────────────────────────────────
def fetch_campaign_list(access_token):
    """All campaigns (id, name, status) — /v3/emails, paginated."""
    print("Fetching campaign list...")
    headers = {"Authorization": f"Bearer {access_token}"}
    campaigns = []
    url = f"{API_BASE}/emails?limit=500"
    while url:
        resp = http_get_json(url, headers)
        campaigns.extend(resp.get("campaigns", []))
        next_link = resp.get("_links", {}).get("next", {}).get("href")
        url = f"{API_ROOT}{next_link}" if next_link else None
    print(f"  → {len(campaigns)} campaigns found")
    return campaigns

def fetch_summary_report(access_token):
    """Bulk performance metrics per campaign activity — up to 500 per page."""
    print("Fetching campaign summary report...")
    headers = {"Authorization": f"Bearer {access_token}"}
    rows = []
    url = f"{API_BASE}/reports/summary_reports/email_campaign_summaries?limit=500"
    while url:
        resp = http_get_json(url, headers)
        rows.extend(resp.get("bulk_email_campaign_summaries", resp.get("results", [])))
        next_link = resp.get("_links", {}).get("next", {}).get("href")
        url = f"{API_ROOT}{next_link}" if next_link else None
    print(f"  → {len(rows)} campaign reports fetched")
    return rows

# ── MERGE ─────────────────────────────────────────────────────────────────────
def pct(n, d):
    return round(n / d * 100, 1) if d else 0.0

def build_campaigns(campaign_info, report_rows):
    campaigns = []
    unmatched = 0
    for row in report_rows:
        cid = row.get("campaign_id")
        info = campaign_info.get(cid)
        if not info:
            unmatched += 1
            info = {"name": "(unknown campaign)", "status": ""}
        stats = row.get("unique_counts", {})
        sends    = stats.get("sends", 0)
        opens    = stats.get("opens", 0)
        clicks   = stats.get("clicks", 0)
        bounces  = stats.get("bounces", 0)
        optouts  = stats.get("optouts", 0)
        forwards = stats.get("forwards", 0)
        abuse    = stats.get("abuse", 0)
        not_opened = stats.get("not_opened", sends - opens if sends else 0)
        campaigns.append({
            "campaign_id":  cid,
            "name":         info["name"],
            "status":       info["status"],
            "sent_date":    (row.get("last_sent_date") or "")[:10],
            "sends":        sends,
            "opens":        opens,
            "clicks":       clicks,
            "bounces":      bounces,
            "optouts":      optouts,
            "forwards":     forwards,
            "abuse":        abuse,
            "not_opened":   not_opened,
            "open_rate":    pct(opens, sends),
            "click_rate":   pct(clicks, sends),
            "bounce_rate":  pct(bounces, sends),
            "optout_rate":  pct(optouts, sends),
        })
    if unmatched:
        print(f"  ⚠ {unmatched} report rows had no matching campaign name (likely deleted/test campaigns)")
    campaigns.sort(key=lambda c: c["sent_date"], reverse=True)
    return campaigns

# ── TOP LINKS ─────────────────────────────────────────────────────────────────
TOP_LINKS_WINDOW_DAYS    = 365  # how far back to consider campaigns
TOP_LINKS_CAMPAIGN_COUNT = 100  # how many highest-clicking campaigns to drill into
TOP_LINKS_RESULT_COUNT   = 30   # how many links to keep in the output

# Mirrors the dashboard's SEP_PATTERN in swapa_email_dashboard.html — keep in sync.
# Matches literal "SEP" but not "SEPT" (September abbreviations in old campaign names).
SEP_PATTERN = re.compile(r"SEP(?!T)")

def clean_link_url(url):
    """
    Constant Contact often reports links wrapped in the recipient's email
    security proxy (e.g. Outlook Safelinks: safelinks.protection.outlook.com/?url=...).
    Unwrap those so the dashboard shows the real destination, not the wrapper.
    """
    try:
        parsed = urllib.parse.urlparse(url)
        if "safelinks.protection.outlook.com" in parsed.netloc:
            qs = urllib.parse.parse_qs(parsed.query)
            if "url" in qs:
                return qs["url"][0]
    except Exception:
        pass
    return url

def fetch_campaign_activity_id(access_token, campaign_id):
    """The clicks report needs a campaign_activity_id, not a campaign_id — look it up."""
    headers = {"Authorization": f"Bearer {access_token}"}
    detail = http_get_json(f"{API_BASE}/emails/{campaign_id}", headers)
    activities = detail.get("campaign_activities", [])
    chosen = next((a for a in activities if a.get("role") == "primary_email"), None) \
             or next((a for a in activities if a.get("role") == "resend"), None)
    return chosen["campaign_activity_id"] if chosen else None

def fetch_click_counts_for_activity(access_token, activity_id):
    """Paginate raw click events for one campaign activity, aggregated by url_id."""
    headers = {"Authorization": f"Bearer {access_token}"}
    counts = {}  # url_id -> {"url": cleaned url, "clicks": n}
    url = f"{API_BASE}/reports/email_reports/{activity_id}/tracking/clicks?limit=500"
    while url:
        resp = http_get_json(url, headers)
        for row in resp.get("tracking_activities", []):
            url_id = row.get("url_id")
            if not url_id:
                continue
            if url_id not in counts:
                counts[url_id] = {"url": clean_link_url(row.get("link_url", "")), "clicks": 0}
            counts[url_id]["clicks"] += 1
        next_link = resp.get("_links", {}).get("next", {}).get("href")
        url = f"{API_ROOT}{next_link}" if next_link else None
    return counts

def fetch_top_links(access_token, campaigns):
    """
    Link-level click detail requires two extra API calls per campaign (activity
    lookup + paginated click events), which doesn't scale to the full ~6,100-campaign
    history. Instead, drill into only the highest-clicking campaigns from the last
    year — a handful of campaigns account for the large majority of total clicks,
    so this captures most of the picture at a small fraction of the API cost.

    Also unconditionally includes every SEP-matching campaign (with clicks > 0),
    regardless of window/rank, so the dashboard's SEP series filter has complete
    link data to show — mirrors SEP_PATTERN used for the campaign-level filter.
    """
    cutoff = (datetime.now(timezone.utc) - timedelta(days=TOP_LINKS_WINDOW_DAYS)).strftime("%Y-%m-%d")
    windowed = [c for c in campaigns if c["sent_date"] >= cutoff and c["clicks"] > 0]
    windowed.sort(key=lambda c: c["clicks"], reverse=True)
    top_by_clicks = windowed[:TOP_LINKS_CAMPAIGN_COUNT]

    sep_campaigns = [c for c in campaigns if c["clicks"] > 0 and SEP_PATTERN.search(c["name"])]

    candidates = list({c["campaign_id"]: c for c in (top_by_clicks + sep_campaigns)}.values())

    print(f"Fetching link-level click detail for {len(candidates)} campaigns "
          f"(top {len(top_by_clicks)} by clicks since {cutoff}, plus {len(sep_campaigns)} SEP campaigns)...")

    link_totals     = defaultdict(lambda: {"url": "", "clicks": 0, "campaign_count": 0})
    link_totals_sep = defaultdict(lambda: {"url": "", "clicks": 0, "campaign_count": 0})
    sep_analyzed = 0
    for i, c in enumerate(candidates, 1):
        is_sep = bool(SEP_PATTERN.search(c["name"]))
        if is_sep:
            sep_analyzed += 1
        try:
            activity_id = fetch_campaign_activity_id(access_token, c["campaign_id"])
            if not activity_id:
                continue
            counts = fetch_click_counts_for_activity(access_token, activity_id)
            for entry in counts.values():
                agg = link_totals[entry["url"]]
                agg["url"] = entry["url"]
                agg["clicks"] += entry["clicks"]
                agg["campaign_count"] += 1
                if is_sep:
                    agg_sep = link_totals_sep[entry["url"]]
                    agg_sep["url"] = entry["url"]
                    agg_sep["clicks"] += entry["clicks"]
                    agg_sep["campaign_count"] += 1
        except Exception as e:
            print(f"  ⚠ skipping campaign {c['campaign_id']} ({c['name']}): {e}")
        if i % 10 == 0:
            print(f"  → {i}/{len(candidates)}")

    links     = sorted(link_totals.values(), key=lambda l: l["clicks"], reverse=True)[:TOP_LINKS_RESULT_COUNT]
    links_sep = sorted(link_totals_sep.values(), key=lambda l: l["clicks"], reverse=True)[:TOP_LINKS_RESULT_COUNT]
    print(f"  → {len(links)} distinct links ranked ({len(links_sep)} SEP-specific)")

    return {
        "window_start":            cutoff,
        "window_end":              datetime.now(timezone.utc).strftime("%Y-%m-%d"),
        "campaigns_analyzed":      len(candidates),
        "campaigns_analyzed_sep":  sep_analyzed,
        "links":                   links,
        "links_sep":               links_sep,
    }

# ── OUTPUT ────────────────────────────────────────────────────────────────────
def write_output(campaigns, top_links):
    OUTPUT_DIR.mkdir(exist_ok=True)
    out_path = OUTPUT_DIR / "email_stats.json"

    sent = [c for c in campaigns if c["sends"] > 0]
    total_sends   = sum(c["sends"] for c in sent)
    total_opens   = sum(c["opens"] for c in sent)
    total_clicks  = sum(c["clicks"] for c in sent)
    total_bounces = sum(c["bounces"] for c in sent)
    total_optouts = sum(c["optouts"] for c in sent)

    payload = {
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "totals": {
            "campaign_count": len(sent),
            "sends":          total_sends,
            "opens":          total_opens,
            "clicks":         total_clicks,
            "bounces":        total_bounces,
            "optouts":        total_optouts,
            "avg_open_rate":  pct(total_opens, total_sends),
            "avg_click_rate": pct(total_clicks, total_sends),
            "avg_bounce_rate": pct(total_bounces, total_sends),
        },
        "campaigns": campaigns,
        "top_links": top_links,
    }

    with open(out_path, "w") as f:
        json.dump(payload, f, indent=2)

    print(f"\n✓ Written to {out_path}")
    print(f"  {len(sent)} sent campaigns, {total_sends:,} total sends, "
          f"{payload['totals']['avg_open_rate']}% avg open rate, "
          f"{payload['totals']['avg_click_rate']}% avg click rate")

# ── MAIN ──────────────────────────────────────────────────────────────────────
def main():
    access_token = get_access_token()
    campaigns = fetch_campaign_list(access_token)
    campaign_info = {
        c["campaign_id"]: {"name": c.get("name", ""), "status": c.get("current_status", "")}
        for c in campaigns
    }
    report_rows = fetch_summary_report(access_token)
    merged = build_campaigns(campaign_info, report_rows)
    top_links = fetch_top_links(access_token, merged)
    write_output(merged, top_links)

if __name__ == "__main__":
    main()