"""Empirical test: how standardised are cash-flow tags across the Mag 7?
Sections are assigned by anchoring on the three us-gaap subtotal rows, which is
robust across filers (section *header* abstracts are not)."""
from edgar import *
set_identity("Your Name your@email.com")
import pandas as pd, json

T={"Apple":"AAPL","Microsoft":"MSFT","Alphabet":"GOOGL","Amazon":"AMZN","Nvidia":"NVDA","Meta":"META","Tesla":"TSLA"}
SUB={"operating":"NetCashProvidedByUsedInOperatingActivities",
     "investing":"NetCashProvidedByUsedInInvestingActivities",
     "financing":"NetCashProvidedByUsedInFinancingActivities"}

def base(c): return c.split("_",1)[1] if "_" in c else c

out={}
for name,tk in T.items():
    f = Company(tk).get_filings(form="10-Q").latest(1)
    df = f.xbrl().statements.cashflow_statement().to_dataframe().reset_index(drop=True)
    df["tag"]=df.concept.str.replace("_",":",n=1)
    df["is_custom"]=~df.tag.str.startswith("us-gaap:")
    df["is_abstract"]=df.concept.str.endswith("Abstract")
    df["b"]=df.concept.map(base)

    idx={}
    for sec,tag in SUB.items():
        hits=df.index[(df.b==tag) & (~df.is_abstract)].tolist()
        idx[sec]=hits[0] if hits else None
    sec_of=[]
    for i in df.index:
        s="other"
        if idx["operating"] is not None and i<=idx["operating"]: s="operating"
        elif idx["investing"] is not None and i<=idx["investing"]: s="investing"
        elif idx["financing"] is not None and i<=idx["financing"]: s="financing"
        sec_of.append(s)
    df["section"]=sec_of
    real=df[~df.is_abstract].copy()

    out[name]={
      "ticker":tk, "accession":f.accession_no, "filed":str(f.filing_date),
      "line_items":int(len(real)),
      "custom_tags":int(real.is_custom.sum()),
      "pct_us_gaap": round(100*(1-real.is_custom.mean()),1),
      "custom_list":sorted(real[real.is_custom].tag.unique().tolist()),
      "subtotals_all_standard": all(v is not None for v in idx.values()),
      "by_section":{s:int((real.section==s).sum()) for s in ["operating","investing","financing"]},
      "financing_items":[[r.tag,r.label] for _,r in real[real.section=="financing"].iterrows()],
      "investing_items":[[r.tag,r.label] for _,r in real[real.section=="investing"].iterrows()],
    }
    o=out[name]
    print(f"{name:<10} items={o['line_items']:>3} custom={o['custom_tags']:>2} ({o['pct_us_gaap']:>5.1f}% us-gaap)  "
          f"op/inv/fin={o['by_section']['operating']}/{o['by_section']['investing']}/{o['by_section']['financing']}  "
          f"subtotals_standard={o['subtotals_all_standard']}")

allc=sorted({c for v in out.values() for c in v["custom_list"]})
print(f"\nTotal distinct custom extension tags across the seven: {len(allc)}")
for c in allc: print("   ",c)
json.dump(out, open("data/tag_standardization.json","w"), indent=1)
print("\nwrote data/tag_standardization.json")