"""
resco_trace.py -- mitmproxy addon for Resco / Dynamics troubleshooting captures.

Does two jobs at once (load this one file, get both):

  1. REDACTION -- masks credentials so a capture is safe to send to support:
       - Authorization / Proxy-Authorization / Cookie headers on every request
         (the Bearer OAuth token on Dynamics calls, the Basic username:password
          on on-prem rescocrm.com calls)
       - client_secret / assertion / code / password / refresh_token in the
         bodies of requests to Microsoft OAuth token endpoints

  2. SUMMARY -- writes "sync-summary.html" next to this file: a one-page,
     pre-analyzed report of the capture so support doesn't have to read
     1000+ raw flows. It shows:
       - time spent per entity (which data table is your bottleneck)
       - the slowest single request per entity
       - every request that returned an error (status 400+)
       - identical requests that were sent more than once

How to load it (no command line needed):
  mitmweb UI -> Options tab -> "scripts" -> full path to this file.
  Or just use Start-Capture.bat, which loads it for you.

The report refreshes every couple of seconds while traffic flows and is
finalized when mitmweb closes. Open sync-summary.html in any browser.

Safety note (why this never breaks live traffic):
  All work happens in the `response` hook. By then mitmproxy has already
  sent the real request to the server, so editing the stored request here
  only changes the saved copy -- the live call keeps its real credentials
  and works normally. Nothing in the live response is altered.

  It does NOT scrub the token that the identity provider returns in the
  *response* to sign-in (scrubbing that live would break login). To keep
  that out of a handoff, sign in first, then clear the flow list, then run
  the sync you want to capture.
"""

import os
import re
import html
import time
import hashlib
from collections import defaultdict
from typing import Sequence

from mitmproxy import http, ctx, flowfilter, command


# ===================== RESCO CAPTURE CONFIG =====================
# Hosts we intercept, matched as regular expressions against "host:port".
# This drives mitmproxy's built-in `allow_hosts`, so everything NOT on this
# list passes through untouched -- no decryption, no cert prompts, less noise,
# and a much smaller capture. Curate this for your environment; it is also
# editable live in the mitmweb Options panel as `_resco_hosts`.
#
# Note: the Microsoft login hosts are included so AUTH problems can be traced.
# That means sign-in token *responses* land in the capture (request-side creds
# are redacted). If you only care about sync, you can drop the login hosts.
DEFAULT_RESCO_HOSTS = [
    "dynamics\\.com",               # Dynamics 365 / Dataverse
    "rescocrm\\.com",               # Resco on-prem / cloud CRM
    "resco\\.net",                  # Resco services / licensing
    "login\\.microsoftonline\\.com",  # Microsoft Entra / OAuth token endpoint
    "login\\.windows\\.net",
    "login\\.live\\.com",
]

# Safety cap: keep at most this many recent flows in memory for the report and
# the HAR export. Older flows roll off (a banner in the report notes when this
# happens). Prevents a capture left running all day from growing without bound.
_MAX_FLOWS = 20000


# ===================== REDACTION CONFIG =====================
SECRET_HEADERS = ("authorization", "proxy-authorization", "cookie")
TOKEN_HOST_HINTS = ("login.microsoftonline.com", "login.windows.net", "login.live.com")
TOKEN_PATH_HINTS = ("/oauth2/", "/token")
MASK = "***REDACTED***"

# Patterns used to figure out which entity a request touches.
# NOTE: these are deliberately anchored. The earlier versions matched too
# loosely and picked up nested attributes (ParentEntityName, LinkEntityName)
# and link-entity aliases, producing bogus labels like "l0" or
# "p_resco_questionnaire1". The fixes below:
#   - `<Entity\b ... \bEntityName` requires a real <Entity> tag AND a word
#     boundary before EntityName, so "ParentEntityName=" no longer matches.
#   - `[^>]*?` keeps the match inside the one opening tag (can't cross '>').
#   - the FetchXML pattern requires "<entity", so "<link-entity" is excluded.
_ENTITY_PATTERNS = (
    re.compile(r'<Entity\b[^>]*?\bEntityName\s*=\s*["\']([^"\']+)["\']', re.I),  # Resco REST/OData
    re.compile(r'<entity\s+name\s*=\s*["\']([^"\']+)["\']', re.I),               # FetchXML primary entity
    re.compile(r'"LogicalName"\s*:\s*"([^"]+)"', re.I),                          # SOAP / JSON
)


def _is_token_endpoint(flow: http.HTTPFlow) -> bool:
    host = flow.request.pretty_host.lower()
    path = flow.request.path.lower()
    if any(h in host for h in TOKEN_HOST_HINTS):
        return True
    return any(p in path for p in TOKEN_PATH_HINTS)


def _scrub_headers(message) -> None:
    for name in SECRET_HEADERS:
        if name in message.headers:
            message.headers[name] = MASK


def _scrub_form_secrets(text: str) -> str:
    return re.sub(
        r"((?:client_secret|client_assertion|assertion|code|password|refresh_token)=)[^&\s]+",
        r"\1" + MASK,
        text,
        flags=re.IGNORECASE,
    )


def _entity_from_text(text: str) -> str:
    if not text:
        return "(no body)"
    for pat in _ENTITY_PATTERNS:
        m = pat.search(text)
        if m:
            return m.group(1).lower()
    return "(unknown)"


# --- Aggregate / "Sync Analyzer" query detection ---
# These are FetchXML count/aggregate queries, marked by aggregate="true" on the
# <fetch> element (with per-attribute aggregate="count"/"sum"/... and an alias).
# They are meta-queries (they count records, they don't transfer them), so we
# pull them out of the per-entity timing and list them separately.
_AGG_RE = re.compile(r'aggregate\s*=\s*["\']true["\']', re.I)
_ALIAS_RE = re.compile(r'alias\s*=\s*["\']([^"\']+)["\']', re.I)


def _is_aggregate(text: str) -> bool:
    return bool(_AGG_RE.search(text))


def _extract_count(alias: str, resp_text: str):
    """Best-effort: pull the aggregate result (the record count) out of the
    response. Returns an int or None. NOTE: the exact response shape hasn't
    been verified against real data yet -- see the note in the chat. If counts
    come back blank/wrong, one sanitized aggregate response fixes this."""
    if not resp_text:
        return None
    # 1) Value keyed by the alias (element or JSON forms)
    if alias:
        a = re.escape(alias)
        for pat in (rf"<{a}\b[^>]*>\s*([0-9]+)", rf'"{a}"\s*:\s*"?([0-9]+)'):
            m = re.search(pat, resp_text, re.I)
            if m:
                return int(m.group(1))
    # 2) Dynamics SOAP KeyValuePair: <b:key>cnt</b:key>...<b:value...>N</b:value>
    if alias:
        m = re.search(
            re.escape(alias) + r"</[^>]*key>.*?<[^>]*value[^>]*>\s*([0-9]+)",
            resp_text,
            re.I | re.S,
        )
        if m:
            return int(m.group(1))
    return None


# Best-effort count of records inside a DATA response (not an aggregate query),
# used to estimate how many records were downloaded per entity. Dynamics wraps
# each returned record as <a:Entity>, Resco REST as <Entity ...>. Formatted-value
# lookups can also appear as nested <Entity>, so this can OVER-count until the
# pattern is confirmed against a real response -- adjust _RECORD_RE if the totals
# look wrong. It returns 0 when nothing matches (so a wrong pattern is obvious).
_RECORD_RE = re.compile(r"<(?:\w+:)?Entity[\s/>]", re.I)


def _count_records(resp_text: str) -> int:
    if not resp_text:
        return 0
    return len(_RECORD_RE.findall(resp_text))


# ===================== FORMATTING HELPERS =====================
def _ms(v: float) -> str:
    return f"{v:,.0f} ms" if v < 1000 else f"{v / 1000:,.1f} s"


def _size(v: float) -> str:
    for unit in ("B", "KB", "MB"):
        if v < 1024:
            return f"{v:,.0f} {unit}"
        v /= 1024
    return f"{v:,.1f} GB"


def _esc(s) -> str:
    return html.escape(str(s))


def _full_url(u: str) -> str:
    return u  # URLs are shown in full; long ones wrap via CSS (word-break)


# Max rows in the searchable detail table (keeps the file sane on huge captures).
_DETAIL_CAP = 8000

# Max bars drawn in the timeline waterfall.
_TIMELINE_CAP = 3000

# Drill-down (_resco_details): how much request/response body to embed in the
# report. Per-body truncation, and a total budget across the whole report so a
# huge capture can't balloon the HTML. Request payloads are inlined for every
# row; response bodies only for errors (status >= 400).
_DETAIL_BODY_CAP = 16 * 1024        # per body, bytes
_DETAIL_TOTAL_CAP = 8 * 1024 * 1024  # total inlined across the report, bytes

_CSS = """
  body { font-family: -apple-system, Segoe UI, Roboto, sans-serif; margin: 2rem;
         color: #1a1a1a; max-width: 1100px; }
  h1 { font-size: 1.4rem; margin-bottom: .2rem; }
  h2 { font-size: 1.05rem; margin-top: 1.8rem; border-bottom: 2px solid #eee;
       padding-bottom: .3rem; }
  details.sec { margin-top: 1.5rem; }
  summary.sec-h { font-size: 1.05rem; font-weight: 600; cursor: pointer;
                  border-bottom: 2px solid #eee; padding-bottom: .3rem;
                  list-style: none; outline: none; }
  summary.sec-h::-webkit-details-marker { display: none; }
  summary.sec-h::before { content: "\u25BE "; color: #999; font-size: .8rem; }
  details:not([open]) > summary.sec-h::before { content: "\u25B8 "; }
  summary.sec-h span { font-weight: 400; color: #888; }
  details.sec > *:nth-child(2) { margin-top: .7rem; }
  .meta { color: #666; font-size: .85rem; margin-bottom: 1.2rem; }
  .cards { display: flex; gap: 1rem; flex-wrap: wrap; }
  .card { background: #f6f7f9; border-radius: 8px; padding: .8rem 1.1rem; }
  .card .big { font-size: 1.5rem; font-weight: 600; }
  .card .lbl { color: #666; font-size: .8rem; }
  table { border-collapse: collapse; width: 100%; font-size: .88rem; margin-top: .5rem; }
  th, td { text-align: left; padding: .35rem .6rem; border-bottom: 1px solid #eee;
           vertical-align: top; }
  th { color: #555; font-weight: 600; }
  td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
  td.url { color: #333; font-family: ui-monospace, Consolas, monospace;
           font-size: .8rem; word-break: break-all; }
  tr.err td { background: #fdecec; }
  tr.err td.num:nth-child(4), tr.err td.num:first-child { color: #b00020; font-weight: 600; }
  tr.entrow { cursor: pointer; }
  tr.entrow:hover td { background: #eef4ff; }
  .ok { color: #1a7f37; }
  .hint { color: #666; font-size: .82rem; margin-top: .4rem; }
  details.drill { margin-top: .35rem; }
  details.drill > summary { cursor: pointer; color: #4a7fd0; font-size: .78rem;
       font-family: -apple-system, Segoe UI, sans-serif; list-style: none; }
  details.drill > summary::-webkit-details-marker { display: none; }
  details.drill > summary::before { content: "▸ payload"; }
  details.drill[open] > summary::before { content: "▾ payload"; }
  .bodylabel { font-size: .72rem; color: #666; margin: .45rem 0 .15rem;
       font-family: -apple-system, Segoe UI, sans-serif; font-weight: 600; }
  pre.body { white-space: pre-wrap; word-break: break-word; background: #fbfbfc;
       border: 1px solid #e5e5e5; border-radius: 4px; padding: .5rem .6rem;
       font-size: .74rem; max-height: 24rem; overflow: auto; margin: 0; }
  .controls { margin: .6rem 0; display: flex; gap: .8rem; align-items: center;
              flex-wrap: wrap; }
  #q { padding: .45rem .6rem; font-size: .9rem; width: 22rem; max-width: 100%;
       border: 1px solid #ccc; border-radius: 6px; }
  .controls label { font-size: .85rem; color: #444; }
  #count { color: #666; font-size: .82rem; }
  table.sortable th { cursor: pointer; user-select: none; }
  table.sortable th:hover { color: #000; }
  .arr { color: #888; font-size: .72rem; }
  .tl-labels { position: relative; height: 1.1rem; font-size: .72rem; color: #888;
               margin-top: .4rem; }
  .tl-labels span { position: absolute; transform: translateX(-50%); white-space: nowrap; }
  .tl-rows { position: relative; border: 1px solid #eee; border-radius: 4px;
             max-height: 70vh; overflow-y: auto;
             background: repeating-linear-gradient(to right,
               transparent 0, transparent calc(10% - 1px), #f0f0f0 10%); }
  .tl-row { position: relative; height: var(--rowh); }
  .tl-bar { position: absolute; top: 0; height: calc(var(--rowh) - 1px);
            border-radius: 1px; min-width: 3px; display: block; }
  a.tl-bar { cursor: pointer; text-decoration: none; }
  a.tl-bar:hover { outline: 1px solid #333; outline-offset: 1px; }
  .tl-bar.data { background: #4a7fd0; }
  .tl-bar.agg  { background: #12a594; }
  .tl-bar.annbody { background: #8b3fc7; }
  .tl-bar.err  { background: #c0271f; }
  tr.flash td { background: #fff3cd !important; transition: background .3s ease; }
  .legend { font-size: .78rem; color: #555; margin: .5rem 0; display: flex;
            gap: 1.1rem; flex-wrap: wrap; }
  .legend i { display: inline-block; width: .8rem; height: .8rem; border-radius: 2px;
              vertical-align: -1px; margin-right: .3rem; }
"""

# Plain (non f-string) so JS braces don't collide with Python formatting.
_JS = """
<script>
(function () {
  var q = document.getElementById('q');
  var eo = document.getElementById('erronly');
  var ex = document.getElementById('exact');
  var rows = Array.prototype.slice.call(document.querySelectorAll('#detail tbody tr'));
  var tlRows = Array.prototype.slice.call(document.querySelectorAll('#tlrows .tl-row'));
  var countEl = document.getElementById('count');
  function apply() {
    var t = q.value.toLowerCase().trim();
    var errOnly = eo.checked;
    var exact = ex && ex.checked;
    function ok(el) {
      var okText;
      if (t === '') {
        okText = true;
      } else if (exact) {
        okText = (el.getAttribute('data-entity') || '').toLowerCase() === t;
      } else {
        var hay = el.getAttribute('data-text');
        if (hay === null) hay = el.textContent;
        okText = hay.toLowerCase().indexOf(t) !== -1;
      }
      var okErr = !errOnly || el.getAttribute('data-err') === '1';
      return okText && okErr;
    }
    var shown = 0;
    for (var i = 0; i < rows.length; i++) {
      var vis = ok(rows[i]);
      rows[i].style.display = vis ? '' : 'none';
      if (vis) shown++;
    }
    // Timeline follows the same filter; hidden rows collapse so it stays compact.
    for (var k = 0; k < tlRows.length; k++) {
      tlRows[k].style.display = ok(tlRows[k]) ? '' : 'none';
    }
    if (countEl) countEl.textContent = 'Showing ' + shown + ' of ' + rows.length;
  }
  if (q) q.addEventListener('input', apply);
  if (eo) eo.addEventListener('change', apply);
  if (ex) ex.addEventListener('change', apply);
  // Click an entity row to filter the detail table to exactly that entity.
  var ent = document.querySelectorAll('tr.entrow');
  for (var j = 0; j < ent.length; j++) {
    (function (el) {
      el.addEventListener('click', function () {
        if (!q) return;
        q.value = el.getAttribute('data-entity') || '';
        if (ex) ex.checked = true;   // exact, so "account" != "stago_accountcategory"
        apply();
        var d = document.getElementById('detail');
        if (d) d.scrollIntoView({ behavior: 'smooth' });
      });
    })(ent[j]);
  }
  apply();

  // Column sorting for any <table class="sortable"> (numeric via data-sort).
  function cellVal(cell) {
    if (!cell) return '';
    var s = cell.getAttribute('data-sort');
    if (s !== null) { var n = parseFloat(s); return isNaN(n) ? s.toLowerCase() : n; }
    return cell.textContent.trim().toLowerCase();
  }
  function makeSortable(table) {
    if (!table.tHead) return;
    var ths = table.tHead.rows[0].cells;
    for (var c = 0; c < ths.length; c++) {
      (function (col) {
        ths[col].addEventListener('click', function () {
          var tb = table.tBodies[0];
          if (!tb) return;
          var rows = Array.prototype.slice.call(tb.rows);
          var asc = ths[col].getAttribute('data-dir') !== 'asc';
          for (var k = 0; k < ths.length; k++) {
            ths[k].removeAttribute('data-dir');
            var old = ths[k].querySelector('.arr');
            if (old) old.remove();
          }
          ths[col].setAttribute('data-dir', asc ? 'asc' : 'desc');
          rows.sort(function (x, y) {
            var xa = cellVal(x.cells[col]), ya = cellVal(y.cells[col]);
            if (xa < ya) return asc ? -1 : 1;
            if (xa > ya) return asc ? 1 : -1;
            return 0;
          });
          for (var i = 0; i < rows.length; i++) tb.appendChild(rows[i]);
          var arr = document.createElement('span');
          arr.className = 'arr';
          arr.textContent = asc ? ' \u25B2' : ' \u25BC';
          ths[col].appendChild(arr);
        });
      })(c);
    }
  }
  var st = document.querySelectorAll('table.sortable');
  for (var s = 0; s < st.length; s++) makeSortable(st[s]);

  // Timeline bar -> jump to its request row and open its payload drill-down.
  var tbars = document.querySelectorAll('a.tl-bar[data-target]');
  for (var b = 0; b < tbars.length; b++) {
    tbars[b].addEventListener('click', function (e) {
      e.preventDefault();
      var row = document.getElementById(this.getAttribute('data-target'));
      if (!row) return;
      row.style.display = '';                 // reveal even if filtered out
      var d = row.querySelector('details.drill');
      if (d) d.open = true;
      row.scrollIntoView({ behavior: 'smooth', block: 'center' });
      row.classList.add('flash');
      setTimeout(function () { row.classList.remove('flash'); }, 1600);
    });
  }
})();
</script>
"""


# ===================== THE ADDON =====================
class RescoTrace:
    def __init__(self):
        self.records = []
        self.last_write = 0.0
        self._filter = None       # compiled view_filter, or None = record all
        self._filter_src = None   # the raw filter string, for display
        self._trimmed = False     # True once the ring buffer has dropped flows
        try:
            self.base = os.path.dirname(os.path.abspath(__file__))
        except NameError:
            self.base = os.getcwd()

    # -- declare our custom options (shown in the mitmweb Options panel) --
    # Leading underscore keeps them sorted to the very TOP of the alphabetical
    # options list, above mitmproxy's built-ins.
    def load(self, loader) -> None:
        loader.add_option(
            "_resco_hosts", Sequence[str], DEFAULT_RESCO_HOSTS,
            "Resco/Dynamics hosts to intercept (regex, matched against host:port). "
            "Drives allow_hosts. Empty = intercept everything.",
        )
        loader.add_option(
            "_resco_output_dir", str, "",
            "Folder for sync-summary.html and the HAR export. "
            "Empty = the folder this script lives in.",
        )
        loader.add_option(
            "_resco_details", bool, True,
            "Embed request payloads (and error response bodies) in the HTML "
            "report so you can drill into individual requests without the HAR. "
            "Bounded in size; turn off for a metadata-only report.",
        )
        loader.add_option(
            "_resco_formats", str, "html,har",
            "Comma-separated outputs written on stop/save: html, har.",
        )
        loader.add_option(
            "_resco_capture", bool, True,
            "Master capture switch. Turn OFF to pause recording without stopping "
            "the proxy. Prefer the resco.start / resco.stop commands.",
        )

    # -- called at startup and whenever options change --
    def configure(self, updated) -> None:
        # Self-scope: push _resco_hosts into mitmproxy's built-in allow_hosts so
        # the user never has to set it by hand. Setting allow_hosts re-enters
        # configure with only "allow_hosts" in `updated`, so this can't loop.
        if "_resco_hosts" in updated:
            hosts = list(ctx.options._resco_hosts or [])
            if hosts != list(ctx.options.allow_hosts or []):
                ctx.options.update(allow_hosts=hosts)

        # Follow mitmweb's own "view_filter" so the report is scoped to the same
        # flows you see in the UI. Empty filter -> record everything (subject to
        # allow_hosts above).
        if "view_filter" in updated or self._filter_src is None:
            src = getattr(ctx.options, "view_filter", None)
            self._filter_src = src or ""
            if src:
                try:
                    self._filter = flowfilter.parse(src)
                    if self._filter is None:
                        ctx.log.warn("resco_trace.py: empty view_filter parse; recording all")
                except Exception as e:
                    ctx.log.warn(f"resco_trace.py: invalid view_filter '{src}', recording all ({e})")
                    self._filter = None
            else:
                self._filter = None

    def _matches(self, flow: http.HTTPFlow) -> bool:
        if self._filter is None:
            return True
        try:
            return bool(flowfilter.match(self._filter, flow))
        except Exception:
            return True  # never drop a flow because matching errored

    # -- called for every completed request/response --
    def response(self, flow: http.HTTPFlow) -> None:
        # 1) redact ALWAYS -- even flows excluded from the report must be clean
        #    in the exported HAR (the HAR contains every flow).
        _scrub_headers(flow.request)
        if _is_token_endpoint(flow):
            body = flow.request.get_text(strict=False) or ""
            if body:
                flow.request.set_text(_scrub_form_secrets(body))

        # 2) honor the capture switch: when paused we still redact (above) but
        #    record nothing new.
        if not ctx.options._resco_capture:
            return

        # 3) only record flows that match the active view_filter
        if not self._matches(flow):
            return

        # 3) record for the summary
        try:
            self._record(flow)
        except Exception as e:
            ctx.log.warn(f"resco_trace.py: could not record flow: {e}")

        # 4) refresh the report at most every 2 seconds
        now = time.time()
        if now - self.last_write > 2:
            self.last_write = now
            self._write()

    # -- called when mitmproxy shuts down --
    def done(self) -> None:
        try:
            self._save()
            ctx.log.info(f"resco_trace.py: outputs written to {self._outdir()}")
        except Exception as e:
            ctx.log.warn(f"resco_trace.py: could not write outputs: {e}")

    # ===================== COMMANDS (type in the command bar) =====================
    @command.command("resco.start")
    def cmd_start(self) -> str:
        """Clear the current capture and (re)start recording."""
        self._clear()
        ctx.options.update(_resco_capture=True)
        msg = "Resco: capture STARTED (previous flows cleared)."
        ctx.log.warn(msg)
        return msg

    @command.command("resco.stop")
    def cmd_stop(self) -> str:
        """Stop recording and write the report + HAR to the output folder."""
        ctx.options.update(_resco_capture=False)
        self._save()
        msg = f"Resco: capture STOPPED. Output written to {self._outdir()}"
        ctx.log.warn(msg)
        return msg

    @command.command("resco.save")
    def cmd_save(self) -> str:
        """Write the report + HAR now, without changing the capture state."""
        self._save()
        msg = f"Resco: saved. Output in {self._outdir()}"
        ctx.log.warn(msg)
        return msg

    @command.command("resco.reset")
    def cmd_reset(self) -> str:
        """Clear captured flows and the flow list, keep recording."""
        self._clear()
        msg = "Resco: captured flows cleared."
        ctx.log.warn(msg)
        return msg

    @command.command("resco.status")
    def cmd_status(self) -> str:
        """Report current capture state (shown in the Command Result pane)."""
        state = "ON" if ctx.options._resco_capture else "PAUSED"
        note = " (buffer full, oldest dropped)" if self._trimmed else ""
        return (
            f"Resco capture {state} | {len(self.records)} flows{note} | "
            f"hosts={list(ctx.options._resco_hosts)} | out={self._outdir()}"
        )

    # ===================== SAVE / CLEAR HELPERS =====================
    def _outdir(self) -> str:
        d = (ctx.options._resco_output_dir or "").strip() or self.base
        try:
            os.makedirs(d, exist_ok=True)
        except Exception:
            d = self.base
        return d

    def _clear(self) -> None:
        self.records = []
        self._trimmed = False
        # Also clear mitmweb's visible flow list, if that command exists (it
        # does not in mitmdump -- ignore the failure there).
        try:
            ctx.master.commands.call("view.clear")
        except Exception:
            pass

    def _save(self) -> None:
        self._write(with_details=True)  # sync-summary.html (with drill-down)
        fmts = [f.strip().lower() for f in (ctx.options._resco_formats or "").split(",")]
        if "har" in fmts:
            self._write_har()

    def _write_har(self) -> None:
        flows = [r["flow"] for r in self.records if r.get("flow") is not None]
        if not flows:
            return
        path = os.path.join(self._outdir(), "resco-capture.har")
        try:
            ctx.master.commands.call("save.har", flows, path)
        except Exception as e:
            ctx.log.warn(f"resco_trace.py: HAR export failed ({e}); trying native flow file.")
            try:
                alt = os.path.join(self._outdir(), "resco-capture.mitm")
                ctx.master.commands.call("save.file", flows, alt)
            except Exception as e2:
                ctx.log.warn(f"resco_trace.py: flow export also failed: {e2}")

    def _record(self, flow: http.HTTPFlow) -> None:
        resp = flow.response
        if resp is None:
            return
        try:
            dur_ms = (resp.timestamp_end - flow.request.timestamp_start) * 1000
        except Exception:
            dur_ms = 0.0

        body = flow.request.get_text(strict=False) or ""
        req_text = html.unescape(body)  # Dynamics embeds FetchXML HTML-encoded

        if _is_token_endpoint(flow):
            entity, is_agg, count = "(auth)", False, None
            dl_records = 0
        else:
            entity = _entity_from_text(req_text)
            is_agg = _is_aggregate(req_text)
            count = None
            dl_records = 0
            resp_text = resp.get_text(strict=False) or ""
            if is_agg:
                am = _ALIAS_RE.search(req_text)
                alias = am.group(1) if am else None
                count = _extract_count(alias, resp_text)
            else:
                dl_records = _count_records(resp_text)

        # Annotation attachment download: the "annotation" entity fetched with
        # its "documentbody" field. Sync grabs annotation headers first, then
        # the (often large) attachment bodies later -- flag these so the
        # timeline can colour them separately.
        is_ann_body = entity == "annotation" and "documentbody" in req_text.lower()

        dup_key = hashlib.md5(
            (flow.request.method + flow.request.path + body).encode("utf-8", "ignore")
        ).hexdigest()
        try:
            start_ts = float(flow.request.timestamp_start)
        except Exception:
            start_ts = 0.0

        self.records.append(
            {
                "entity": entity,
                "method": flow.request.method,
                "url": flow.request.pretty_url,
                "status": resp.status_code,
                "dur_ms": dur_ms,
                "size": len(resp.raw_content or b""),  # bytes on the wire
                "dup": dup_key,
                "agg": is_agg,       # is this a Sync Analyzer / aggregate query?
                "records": count,    # extracted record count (aggregate queries)
                "dl_records": dl_records,  # records counted in this data response
                "start": start_ts,   # request start (epoch secs), for timeline
                "ann_body": is_ann_body,  # annotation attachment (documentbody) download
                "flow": flow,        # kept for the HAR export (reference, not a copy)
            }
        )

        # Ring buffer: drop the oldest flows once we exceed the cap so a long
        # capture can't grow without bound. We hold references only, so this
        # does not free mitmproxy's own copy -- it just bounds our report/HAR.
        if len(self.records) > _MAX_FLOWS:
            del self.records[: len(self.records) - _MAX_FLOWS]
            self._trimmed = True

    def _timeline_html(self, recs) -> str:
        rows = [r for r in recs if r.get("start")]
        if not rows:
            return "<p class='ok'>No timing data available for a timeline.</p>"
        rows = sorted(rows, key=lambda r: r["start"])
        t0 = rows[0]["start"]
        span_s = max((r["start"] + r["dur_ms"] / 1000.0) for r in rows) - t0
        span_ms = span_s * 1000.0
        if span_ms <= 0:
            span_ms = 1.0

        shown = rows[:_TIMELINE_CAP]
        rowh = 3 if len(shown) > 400 else (6 if len(shown) > 120 else 12)

        bars = []
        for r in shown:
            offset_ms = (r["start"] - t0) * 1000.0
            left = max(0.0, min(100.0, offset_ms / span_ms * 100.0))
            width = max(0.15, r["dur_ms"] / span_ms * 100.0)
            if left + width > 100:
                width = max(0.15, 100 - left)
            if r["status"] >= 400:
                kind = "err"
            elif r.get("ann_body"):
                kind = "annbody"
            elif r.get("agg"):
                kind = "agg"
            else:
                kind = "data"
            label = " · attachment body" if r.get("ann_body") else ""
            tip = (
                f"{r['entity']}{label} · {r['method']} · {r['status']} · "
                f"{_ms(r['dur_ms'])} · +{_ms(offset_ms)}"
            )
            # Same filter attributes as the detail table so one filter drives both.
            haystack = f"{r['entity']} {r['method']} {r['status']} {r['url']}".lower()
            # Link the bar to its row in the detail table (which carries the
            # payload drill-down), when that row exists (idx within the cap).
            idx = r.get("_idx")
            style = f"left:{left:.3f}%;width:{width:.3f}%"
            if idx and idx <= _DETAIL_CAP:
                bar = (f'<a class="tl-bar {kind}" href="#req-{idx}" '
                       f'data-target="req-{idx}" style="{style}" title="{_esc(tip)}"></a>')
            else:
                bar = (f'<div class="tl-bar {kind}" style="{style}" '
                       f'title="{_esc(tip)}"></div>')
            bars.append(
                f'<div class="tl-row" data-entity="{_esc(r["entity"])}" '
                f'data-err="{"1" if r["status"] >= 400 else "0"}" '
                f'data-text="{_esc(haystack)}">'
                f'{bar}</div>'
            )

        # axis labels at 0 / 25 / 50 / 75 / 100 %
        labels = "".join(
            f'<span style="left:{p}%">{_ms(span_ms * p / 100.0)}</span>'
            for p in (0, 25, 50, 75, 100)
        )
        cap_note = (
            f"<p class='hint'>Showing the first {_TIMELINE_CAP:,} of {len(rows):,} "
            f"requests.</p>" if len(rows) > _TIMELINE_CAP else ""
        )
        return (
            f"<p class='hint'>Wall-clock span <b>{_ms(span_ms)}</b> across "
            f"{len(rows):,} requests (bars ordered by start time; overlaps mean "
            f"concurrent requests, gaps mean idle time). Follows the filter in "
            f"<b>All requests</b> below.</p>"
            "<div class='legend'>"
            "<span><i style='background:#4a7fd0'></i>data request</span>"
            "<span><i style='background:#12a594'></i>aggregate / analyzer</span>"
            "<span><i style='background:#8b3fc7'></i>annotation attachment (documentbody)</span>"
            "<span><i style='background:#c0271f'></i>error</span>"
            "<span>hover a bar for details &middot; click to jump to the request</span></div>"
            f"<div class='tl-labels'>{labels}</div>"
            f"<div class='tl-rows' id='tlrows' style='--rowh:{rowh}px'>{''.join(bars)}</div>"
            + cap_note
        )

    def _write(self, with_details: bool = False) -> None:
        recs = self.records
        if not recs:
            return
        # Body drill-down is only built for the final save/stop/done write, not
        # the every-2s live refresh -- extracting bodies for thousands of rows
        # would slow the live report.
        include_details = with_details and bool(ctx.options._resco_details)

        # Stable 1-based index per record, shared by the detail-table row id
        # (req-N) and the timeline bar that links to it.
        for _i, _r in enumerate(recs, 1):
            _r["_idx"] = _i
        total = len(recs)
        total_ms = sum(r["dur_ms"] for r in recs)
        total_bytes = sum(r["size"] for r in recs)
        errors = [r for r in recs if r["status"] >= 400]

        # Sync Analyzer (aggregate/count) queries are meta-queries -- keep them
        # out of the per-entity timing and list them on their own.
        analyzer = [r for r in recs if r.get("agg")]
        data_recs = [r for r in recs if not r.get("agg")]

        # per-entity aggregate (data requests only)
        agg = defaultdict(lambda: {"n": 0, "ms": 0.0, "bytes": 0, "max": 0.0, "recs": 0})
        for r in data_recs:
            a = agg[r["entity"]]
            a["n"] += 1
            a["ms"] += r["dur_ms"]
            a["bytes"] += r["size"]
            a["max"] = max(a["max"], r["dur_ms"])
            a["recs"] += r.get("dl_records", 0)
        entity_rows = sorted(agg.items(), key=lambda kv: kv[1]["ms"], reverse=True)

        # identical requests sent more than once (2xx only, so retries after a
        # 401 don't get counted as wasteful duplicates)
        dup = defaultdict(list)
        for r in recs:
            if 200 <= r["status"] < 300:
                dup[r["dup"]].append(r)
        dup_groups = sorted(
            ([v for v in dup.values() if len(v) >= 2]), key=len, reverse=True
        )[:30]

        # ---- build HTML ----
        # Entity rows: clickable to filter the detail table; each numeric cell
        # carries data-sort so the column sorts numerically, not as text.
        def _erow(name, a):
            recs_n = a.get("recs", 0)
            recs_disp = f"{recs_n:,}" if recs_n else "&mdash;"
            avg = a["ms"] / a["n"] if a["n"] else 0
            return (
                f'<tr class="entrow" data-entity="{_esc(name)}" title="Click to filter the request list below">'
                f"<td>{_esc(name)}</td>"
                f"<td class='num' data-sort='{a['n']}'>{a['n']:,}</td>"
                f"<td class='num' data-sort='{recs_n}'>{recs_disp}</td>"
                f"<td class='num' data-sort='{a['ms']:.3f}'>{_ms(a['ms'])}</td>"
                f"<td class='num' data-sort='{avg:.3f}'>{_ms(avg)}</td>"
                f"<td class='num' data-sort='{a['max']:.3f}'>{_ms(a['max'])}</td>"
                f"<td class='num' data-sort='{a['bytes']}'>{_size(a['bytes'])}</td></tr>"
            )

        rows_html = "".join(_erow(name, a) for name, a in entity_rows)

        # Sync Analyzer (aggregate / count) queries, listed separately.
        def _anrow(r):
            cnt = r["records"]
            cnt_disp = f"{cnt:,}" if cnt is not None else "&mdash;"
            cnt_sort = cnt if cnt is not None else -1
            cls = " class='err'" if r["status"] >= 400 else ""
            return (
                f"<tr{cls}>"
                f"<td>{_esc(r['entity'])}</td>"
                f"<td class='num' data-sort='{cnt_sort}'>{cnt_disp}</td>"
                f"<td class='num' data-sort='{r['dur_ms']:.3f}'>{_ms(r['dur_ms'])}</td>"
                f"<td class='num' data-sort='{r['status']}'>{r['status']}</td>"
                f"<td class='url'>{_esc(_full_url(r['url']))}</td></tr>"
            )

        if analyzer:
            an_sorted = sorted(
                analyzer,
                key=lambda r: (r["records"] if r["records"] is not None else -1),
                reverse=True,
            )
            an_html = "".join(_anrow(r) for r in an_sorted)
            analyzer_section = (
                "<table class='sortable' id='analyzer'><thead><tr>"
                "<th>Entity</th><th>Number of records</th><th>Time</th>"
                "<th>Status</th><th>URL</th></tr></thead><tbody>"
                + an_html + "</tbody></table>"
                "<p class='hint'>&#9888; The record count is read from the "
                "aggregate query response on a best-effort basis and hasn't been "
                "verified against real data yet. If counts look wrong or blank, "
                "one sanitized aggregate response will let me fix the extraction.</p>"
            )
        else:
            analyzer_section = (
                "<p class='ok'>No Sync Analyzer (aggregate) queries were seen.</p>"
            )

        if errors:
            err_html = "".join(
                f"<tr class='err'><td class='num'>{r['status']}</td>"
                f"<td>{_esc(r['entity'])}</td><td>{_esc(r['method'])}</td>"
                f"<td class='num'>{_ms(r['dur_ms'])}</td>"
                f"<td class='url'>{_esc(_full_url(r['url']))}</td></tr>"
                for r in errors[:500]
            )
            err_section = (
                "<table><tr><th>Status</th><th>Entity</th><th>Method</th>"
                "<th>Time</th><th>URL</th></tr>" + err_html + "</table>"
                "<p class='hint'>A 401 that is immediately retried and then "
                "succeeds is the normal sign-in handshake. A 401 or 500 with no "
                "successful retry is what's worth investigating.</p>"
            )
        else:
            err_section = "<p class='ok'>No errors &mdash; every response was 2xx.</p>"

        if dup_groups:
            dup_html = "".join(
                f"<tr><td class='num'>{len(g)}&times;</td>"
                f"<td>{_esc(g[0]['entity'])}</td><td>{_esc(g[0]['method'])}</td>"
                f"<td class='url'>{_esc(_full_url(g[0]['url']))}</td></tr>"
                for g in dup_groups
            )
            dup_section = (
                "<table><tr><th>Count</th><th>Entity</th><th>Method</th>"
                "<th>URL</th></tr>" + dup_html + "</table>"
            )
        else:
            dup_section = "<p class='ok'>No identical requests were repeated.</p>"

        # Drill-down: embed request payload (+ error response body) per row.
        # Bounded by _DETAIL_BODY_CAP per body and _DETAIL_TOTAL_CAP overall.
        det_state = {"budget": _DETAIL_TOTAL_CAP, "capped": False}

        def _body_block(r):
            if not include_details or det_state["budget"] <= 0:
                if include_details:
                    det_state["capped"] = True
                return ""
            flow = r.get("flow")
            if flow is None:
                return ""
            parts = []
            try:
                req = flow.request.get_text(strict=False) or ""
            except Exception:
                req = ""
            if req.strip():
                parts.append(("Request payload", req))
            if r["status"] >= 400 and flow.response is not None:
                try:
                    resp = flow.response.get_text(strict=False) or ""
                except Exception:
                    resp = ""
                if resp.strip():
                    parts.append(("Response body (error)", resp))
            if not parts:
                return ""
            blocks = []
            for label, text in parts:
                truncated = len(text) > _DETAIL_BODY_CAP
                snippet = text[:_DETAIL_BODY_CAP]
                det_state["budget"] -= len(snippet)
                note = " (truncated)" if truncated else ""
                blocks.append(
                    f"<div class='bodylabel'>{_esc(label)}{note}</div>"
                    f"<pre class='body'>{_esc(snippet)}</pre>"
                )
            return "<details class='drill'><summary></summary>" + "".join(blocks) + "</details>"

        # Searchable per-request detail table (full URLs, filterable in-page).
        def _drow(i, r):
            is_err = r["status"] >= 400
            cls = " class='err'" if is_err else ""
            return (
                f'<tr id="req-{i}" data-err="{"1" if is_err else "0"}" '
                f'data-entity="{_esc(r["entity"])}"{cls}>'
                f"<td class='num'>{i}</td>"
                f"<td>{_esc(r['entity'])}</td>"
                f"<td>{_esc(r['method'])}</td>"
                f"<td class='num'>{r['status']}</td>"
                f"<td class='num'>{_ms(r['dur_ms'])}</td>"
                f"<td class='num'>{_size(r['size'])}</td>"
                f"<td class='url'>{_esc(_full_url(r['url']))}{_body_block(r)}</td></tr>"
            )

        detail_rows = "".join(_drow(i, r) for i, r in enumerate(recs[:_DETAIL_CAP], 1))
        cap_note = (
            f"<p class='hint'>Showing the first {_DETAIL_CAP:,} of {total:,} "
            f"requests.</p>" if total > _DETAIL_CAP else ""
        )
        if det_state["capped"]:
            cap_note += (
                "<p class='hint'>&#9888; Payload drill-down reached its size budget; "
                "later rows show metadata only. The full bodies are in the HAR.</p>"
            )

        # ---- Timeline (waterfall) ----
        timeline_section = self._timeline_html(recs)

        generated = time.strftime("%Y-%m-%d %H:%M:%S")
        filter_note = (
            f' &middot; filtered to <code>{_esc(self._filter_src)}</code>'
            if self._filter_src else ""
        )
        body = f"""<h1>Resco sync summary</h1>
<div class="meta">Generated {generated}{filter_note} &middot; refreshes while capturing &middot;
  send this alongside the capture file.</div>

<div class="cards">
  <div class="card"><div class="big">{total:,}</div><div class="lbl">requests</div></div>
  <div class="card"><div class="big">{_ms(total_ms)}</div><div class="lbl">total time</div></div>
  <div class="card"><div class="big">{_size(total_bytes)}</div><div class="lbl">data received</div></div>
  <div class="card"><div class="big">{len(errors):,}</div><div class="lbl">errors (400+)</div></div>
</div>

<details open class="sec"><summary class="sec-h">Time by entity <span>(click a column to sort &middot; click a row to filter below)</span></summary>
<table class="sortable" id="byentity">
  <thead><tr><th>Entity</th><th>Requests</th><th>Records</th><th>Total time</th>
      <th>Avg</th><th>Slowest</th><th>Data</th></tr></thead>
  <tbody>
  {rows_html}
  </tbody>
</table>
<p class="hint">&#9888; "Records" = records counted in the data responses for
  that entity, summed across all its pages. It's a best-effort count and may be
  off until the record pattern is confirmed against a real response. Aggregate
  count-queries are excluded here and listed under Sync Analyzer below.</p>
</details>

<details open class="sec"><summary class="sec-h">Sync Analyzer requests <span>(aggregate / count queries)</span></summary>
{analyzer_section}
</details>

<details open class="sec"><summary class="sec-h">Errors</summary>
{err_section}
</details>

<details open class="sec"><summary class="sec-h">Repeated identical requests</summary>
{dup_section}
</details>

<details open class="sec"><summary class="sec-h">Timeline <span>(request waterfall)</span></summary>
{timeline_section}
</details>

<details open class="sec"><summary class="sec-h">All requests <span>(search / filter)</span></summary>
<div class="controls">
  <input id="q" type="text" placeholder="Type to filter: entity, URL, method, status&hellip;">
  <label><input type="checkbox" id="exact"> Entity exact match</label>
  <label><input type="checkbox" id="erronly"> errors only</label>
  <span id="count"></span>
</div>
{cap_note}
<table id="detail">
  <thead><tr><th>#</th><th>Entity</th><th>Method</th><th>Status</th>
      <th>Time</th><th>Data</th><th>URL</th></tr></thead>
  <tbody>
  {detail_rows}
  </tbody>
</table>
</details>"""

        doc = (
            '<!doctype html><html><head><meta charset="utf-8">'
            "<title>Resco sync summary</title><style>" + _CSS + "</style></head><body>"
            + body + _JS + "</body></html>"
        )

        out_path = os.path.join(self._outdir(), "sync-summary.html")
        with open(out_path, "w", encoding="utf-8") as f:
            f.write(doc)


addons = [RescoTrace()]