iHdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl mZddlmZddlmZmZmZmZmZddlmZddlmZdd lmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z)ddl*m+Z+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3hdZ4edGddZ5hdZ6dRdSdZ7dTdZ8dTd Z9dTd!Z:dUd"Z;dVd#ZdYd&Z?dZd[d'Z@dZd[d(ZAdZd\d)ZBd]d*ZCdXd+ZDd,d-d.d/ d^d0ZEdXd1ZFd_d2ZGd`d3ZHdad4ZIdbd5ZJdcd6ZKddd7ZLdedfd8ZMdad9ZNdgd:ZOdhd;ZPdid<ZQdhd=ZRdjdkd>ZSdld?ZTdmd@ZUdndAZVdXdBZWdXdCZXdldDZYdodEZZdpdFZ[dqdGZ\drdsdHZ]dtdIZ^dudJZ_dvdKZ`dwdLZadxdMZbdydNZcdzdOZdd{dPZeefdQk(r egeey#e $r dZ ddl Z YvwxYw)|aVoucher_List_Scraper.py USAGE pip install selenium # Ensure Firefox + geckodriver are installed. # If geckodriver is not on PATH, set: # export GECKODRIVER_PATH=/full/path/to/geckodriver python3 Voucher_List_Scraper.py python3 Voucher_List_Scraper.py --headless WHAT THIS DOES - Opens the Accounts list page and logs in (credentials hard-coded below). - Clicks id=accSearchBox, types "aa", waits 3 seconds, then presses Enter. - Waits a moment for the results to refresh. - Clicks the FIRST account row in the results table id='device-grid-list'. - Creates a folder in the same directory as this .py. - Writes a Markdown file inside that folder named after the clicked account text. - On the account page, finds id=ProgramListView; if the FIRST program row is active (checkbox checked), it records that program name and opens it. - Records "Program Summary" section (best effort) and captures all rows from the main DataTables product list (not just the first 10 displayed). NOTES - The portal uses DataTables; to avoid stale element errors, this script snapshots tables with JavaScript (reads text + hrefs from the DOM) instead of reading WebElement.text repeatedly. ) annotationsN) dataclass)Path)DictListOptionalSequenceTuple)urljoin) webdriver)TimeoutException)By)Keys) ActionChains)Options)Service)expected_conditions) WebDriverWaitzFhttps://portal.redwingforbusiness.com/RWS_AccountsListPage?tab=accountzwalpine614@rwfb.comzSammiandCoco2!aa accSearchBoxzdevice-grid-listProgramListViewT> town of la platatown of indian headbolling afb commissarycharles county utilitiesbattle creek constructioncharles county public works!charles county emergency services!maryland transportation authority-smithsonian - african american history museum)frozenc"eZdZUded<ded<y)LinkCellstrtexthrefN)__name__ __module__ __qualname____annotations__+Voucher_List_Folder/voucher_scraper_core.pyr#r#^s I Ir,r#>AUXCONNULPRNCOM1COM2COM3COM4COM5COM6COM7COM8COM9LPT1LPT2LPT3LPT4LPT5LPT6LPT7LPT8LPT9c~|xsdjddjddjddj}tjdd|}tjdd|}tjd d|}|j d }|sd }|j t vr|d}|d |S) zReturn a filesystem-safe single path component. - Preserves spaces and '$' (so you can get names like '$150 Subsidy Safety Toe.md'). - Removes characters that are invalid on Windows and path separators on all OSes.     \s+z [<>:"/\\|?*]_z [\x00-\x1f]z .itemN)replacestripresubrstripupper_WINDOWS_RESERVED)smax_lens r-_safe_fs_componentrVks b$$,,T37??cJPPRA vsAA Q'A ~r1%A A wwy%%cG Xg;r,c t|}|jr|jsy|jd5}|j ddk(cdddS#1swYyxYw#t $rYywxYw)zNReturn True if `path` looks like a real ELF binary (not a shell/snap wrapper).FrbsELFN)rexistsis_fileopenread Exception)pathpfs r-_is_elf_executablerbsg Jxxz VVD\ +Q66!9 * + + + s3+A)A)A A)A&"A)&A)) A54A5c t|}|jxr/|jxr|jj dk(S#t $rYywxYw)Nz.exeF)rrZr[suffixlowerr^)r_r`s r-_is_windows_executablerfsN JxxzHaiikHahhnn.>&.HH sA A AAcTtjdk(r t|St|S)Nnt)osnamerfrb)r_s r-_is_usable_executablerks# ww$%d++ d ##r,c2tjjd}|r_tt |j }t |r|Stt |jd}t|r|Stjdk(rFddtjdtjdg}|D]}|st|s|cSyd d g}|D]}t|s|cStjdtjdtjd g}|D]a}|rt|r|cS|st |jdk(s0tt |jd}t|s_|cSy) zFind a real Firefox executable for GeckoDriver. On many Pi / Ubuntu setups, `firefox` on PATH is a snap wrapper script, which GeckoDriver rejects. We prefer `firefox-bin` (real ELF) when available. FIREFOX_BINARYz firefox-binrhz,C:\Program Files\Mozilla Firefox\firefox.exez2C:\Program Files (x86)\Mozilla Firefox\firefox.exez firefox.exefirefoxNz1/snap/firefox/current/usr/lib/firefox/firefox-binz-/snap/firefox/current/usr/lib/firefox/firefoxz firefox-esr) rienvirongetr$r expanduserrk with_namerbrjshutilwhichrf)env_binsibwin_candidatescsnap_candidatespath_candidatess r-_locate_firefox_binaryr{srjjnn-.Gd7m..01  )N$w-))-89 c "J ww$ ; A LL ' LL #     A+A.  <7O a H  ]# Y ]#O  #A&H a*d1g'' 67C!#&  r,cLtjjd}|r/tt |j }t |r|Stjdk(r|r0tt |jd}t|r|Stjdtjdg}|D]}|st|s|cStt tjjdz ddg}|D]}t|s|cSyd}t|r|S|r0tt |jd}t|r|Stjd}|r t|r|Sgd }|D]}t|s|cSy) z))+225FFGH  ,UG45 E5XF F Z[*(++  E58D D E *))*s$ A5) B5B BB)(B)cL ttjj}|dz }|j ddt j jdt|t j jdt|dz y#t$rYywxYw)zIUse a project-local Selenium cache to avoid unwritable default locations.z.selenium-cacheTparentsexist_ok SE_CACHE_PATHXDG_CACHE_HOMEz.cacheN) rrrrmkdirriro setdefaultr$r^)base cache_dirs r-_configure_selenium_runtime_envr(s H~%%'..,, t4 os9~> .D8O0DE    sBB B#"B#cjtt}|r|jdt}|rt d|||_n t d|j dd|j dd tjt||} |jdd|S#t$rjtjd k(rUt d t d |rd nd t d|xsdt dt|rd nd t dwxYw#t$rY|SwxYw)N --headlessz[driver] Using firefox binary: zf[driver] WARNING: Could not locate a real Firefox binary; GeckoDriver may fail on snap-wrapper setups.zdom.webnotifications.enabledFzmedia.volume_scalez0.0)serviceoptionsrhz[driver] Windows setup check:z[driver] Firefox detected: yesnoz[driver] Firefox path: z (not found)z![driver] geckodriver detected: zT[driver] Tip: install geckodriver.exe and put it on PATH, or set GECKODRIVER_PATH.ii)rr add_argumentr{rbinary_locationset_preferencer Firefoxrr^rirjrset_window_size)headlessoptsrdrivers r- build_driverr4sA#% 9D ,')*K / }=>* vw 6>,e4 ""+? +LVZ[ tS) M  77d? 1 2 1;%D1QR S -k.J]-KL M 51+>UDIK L h i    M s: B/D%/A3D"% D21D2cbt||jtj||fSN)runtilECpresence_of_element_locatedrbyvaluetimeouts r- wait_presentrWs+  ) / /0N0NPRTY{0[ \\r,cbt||jtj||fSr)rrrelement_to_be_clickablers r-wait_clickabler[s*  ) / /0J0JBPU;0W XXr,cJtj|z}tj|krf |jtjd}|syt d|Dsy tj dtj|kreyy#t $rYywxYw)z@Best-effort wait for DataTables processing overlay to disappear.zdiv.dataTables_processingNc3<K|]}|jywr) is_displayed).0r`s r- z'datatables_wait_idle..gs7Aq~~'7sg?)time find_elementsr CSS_SELECTORanyr^sleep)rrendprocss r-datatables_wait_idler_s ))+ C ))+  ((:UVE7778 3 ))+    s"BB B"!B"c*d}|j||S)zPReturn headers + row cell texts + first link (text+href) per row for a table id.as const table = document.getElementById(arguments[0]); if (!table) return null; const headers = Array.from(table.querySelectorAll('thead th')).map(th => (th.innerText || '').trim()); const rows = Array.from(table.querySelectorAll('tbody tr')).map(tr => { const cells = Array.from(tr.querySelectorAll('td')).map(td => (td.innerText || '').trim()); const a = tr.querySelector('a[href]'); const link = a ? {text: (a.innerText||'').trim(), href: a.getAttribute('href')||''} : null; return {cells, link}; }); return {headers, rows}; )execute_scriptrtable_idscripts r-js_table_snapshotrns F   22r,c|jtt|tjdd}t|tjdd} |j d| |j|jt |j|jt |jtjdjt|tjtdy#t $rYwxYw#t $rYwxYw#t $rYwxYw#t $r"|jtjYxwxYw)Nusernamepasswordz.arguments[0].scrollIntoView({block:'center'});Login<)rp LOGIN_URLrrIDrr^clear send_keysUSERNAMEPASSWORD find_elementclickrENTERACC_SEARCHBOX_ID)ruser_elpass_els r-loginrs JJy6255*b9G6255*b9G NPWX   h   h&BEE7+113  0"5)            &$**%&sHC="D D..D*= D D  DD D'&D'*(EE@??pre_enter_sleeppost_enter_sleep final_sleepcL|xsdjj}td|dt|tj t d}|j |j|j|tj| |j |jtj  |jtj" t%|jtj j' |jtj(tj|t+|dtj|y#t$rT |jtjd|jtjn#t$rYnwxYwYawxYw#t$rY:wxYw#t$rY*wxYw#t$rYwxYw#t$rYwxYw#t$rYwxYw)z6Type `term` into accSearchBox, wait, then press Enter.rEzFiltering accounts: typing 'z' into accSearchBox...raN)rNrerrrrrrrr^rrCONTROL BACKSPACErrrRETURNrperformTABr)rtermrrrboxs r-filter_accounts_termrs JB    % % 'D (.D EF (8" =CIIK MM$JJ    djj!  dkk" V&&tzz2::<  dhh JJ $JJ{K   MM$,, , MM$.. )                      s#E8G+G( G8+6H"H8 G?GG G G GGG G%$G%( G54G58 HH HH H#"H#c"t|tS)z&Back-compat wrapper: uses SEARCH_TERM.)r SEARCH_TERM)rs r-filter_accounts_simplers  44r,ct|tjtddd}t |dj |t |t}|r|jds td|jdxsgDcgc]}|}}|dd}|jd}|r|jds td |jd xsg}i}t|D]#\} }| t|ks|| ||xsd | <%t|jd d j|jdd j|fScc}w)zHReturn the first account link + row mapping from accounts results table.rc |jtjt}|j tj d}t |dkDS#t$rYywxYw)Nztbody tr td a[href]rF)rrrACCOUNTS_TABLE_IDrrlenr^)dtblrs r-_has_first_linkz7get_first_account_from_results.._has_first_linksS ..(9:C!!"//3HIAq6A:   sAA A A rowsz)Accounts results table snapshot is empty.headersrlinkr&z9Could not find a clickable account link in the first row.cellscol_r%rEr%r&)returnbool) rrrrrrrrp RuntimeError enumeraterr#rN) rrsnaphrfirstrrrow_mapis r-get_first_account_from_resultsrsF 126&"##O4 V%6 7D txx'FGG&*hhy&9&?RA!AGA LOE 99V D txx'VWWyy)/RE G'"01 s5z>',QxGAO4s $0 &"-335DHHVRCOMPANY ADDRESSTAX INFORMATIONCONTACT INFORMATIONCREDIT & BILLING INFORMATION#PAYMENT TERMS & INVOICE PREFERENCESACCOUNTSPROGRAMS ATTACHMENTSPcp|j}tD]\}}|j}||k(rKt|dztD]0}|j }|s|jdvr,|ccS|j |dzs{|t|dj cSy)Nr > ACCOUNT # COMPANY NAMEACCOUNT NUMBERPARENT ACCOUNTrGrE)rRrrangerrN startswith)labellab_uidxrawruknxtsections r- _value_afterz9parse_company_information_from_text.._value_after%s !'* 0HCBU{sQwG 5A!!***,C yy{&gg J }}US[)3u:;'--// 0r,rparent_accountr company_namerraccount_number)rr$rr$) splitlinesrNrRrrrappenditems) page_textlnlinesrRstartru stop_headersjr#outr vr"s @r-#parse_company_information_from_textr3sZ$-?">">"@ ABRXXZ AE A"' (BRXXZ (E ( E% 1 % %)>!)CEE  {  LG 5#e* % 8| # uQx w<2    C()9:C&~6C()9:Wl;>WC YY[ .TQAAqD ..e B (b /sD!D& D+D+c t|tjdd|jtjd}|jxsdj S#t $rYywxYw)zGet the account name from the Account Summary header span. After clicking an account row, the portal shows the canonical account name in: Aaron Equipment Company z AccountSummary:accountForm:aname rE)rrrrr%rNr^)rels r-"get_account_name_from_summary_spanr7=sZ VRUU$FK  (J K 2$$&& sAA A('A(c |ddh}g}t|xsgD]@\}}|xsdj}|s|j|vr0|j|B|s||fS|Dcgc]}|| }}g}|xsgD]4} |j|Dcgc]}|t | kr| |ndc}6||fScc}wcc}w)aDrop backend/hidden columns that DataTables sometimes includes in the DOM. The Red Wing portal product table often contains extra non-user columns such as: - Product Id - Filter Number These may be invisible in the UI but appear in our scrape output. z product idz filter numberrE)rrNrer(r) rr drop_nameskeep_idxrrhh new_headersnew_rowsrs r-drop_hidden_product_columnsr?Ks0JH'-R(1g2__   88: #  }'/0!71:0K0 "H ZRJxH!!c!f*!A$"4HIJ   1Is # B4 B9 ct|tjtdt |tt j dt|dt|tg}t}tdD]t}t|t}|r|jds|S|jdxsgDcgc]}|}}|jdxsgD]}|jdxsi}|jdxsd j} | r| |vrB|j| t|jd xsd j| } |jd xsg} i} t!|D]#\} }| t#| ks| | | |xsd | <%|j%| | ft'|t}|s|St j dt|dw|Scc}w)zHReturn ALL account links + row mappings from the accounts results table.r皙?ri,rrrr&rEr%rrrffffff?)rrrrset_datatable_length_allrrrdatatable_goto_firstsetrrrprNaddr#rrr(datatable_click_next)rr1seenrKrrrrowrr& link_cellrrrclickeds r-get_all_accounts_from_resultsrLis 126V%67JJsO$!2313C 5D 3Z) ):;488F+ 0 J-+/((9*=*CEAaEE88F#)r -C776?(bDHHV$*113D44< HHTN txx'7'=2&D&D&FTRI"www/52E&(G!'* 81s5z>/4QxGAO4s, 8 JJ 7+ , -'v/@A  J 4VR(3)6 J-Fs: G3cld} |j||}t|xsgS#t$rgcYSwxYw)z>Snapshot program rows: link text/href + checkbox active state.aF const table = document.getElementById(arguments[0]); if (!table) return []; const rows = Array.from(table.querySelectorAll('tbody tr')).map(tr => { const a = tr.querySelector('a[href]'); if (!a) return null; const cb = tr.querySelector('input[type="checkbox"]'); const active = cb ? (cb.checked || cb.getAttribute('checked') !== null) : false; return {text: (a.innerText||'').trim(), href: a.getAttribute('href')||'', active: active}; }).filter(x => x && x.href); return rows; )rlistr^)rrrress r-js_program_list_snapshotrPsB F##FH5CI2  s % 33c t|tjtdt |tt jdt|dt|tg}t}tdD]}t|t}|D]}|jdxsdj}|r||vr-|j|t!|jd}|r|s]|j#|jdxsdj||d t%|t} | s|St jd t|d|S#t$rgcYSwxYw) zuReturn program links from the 'List Of Account Related Programs' table. Returns dicts: {text, href, active} rrArr&rEactiver%)r%r&rSrB)rrrPROGRAM_TABLE_IDr rCrrrrDrErrPrprNrFrr(rG) r only_activer1rHrKrr>r&rSrKs r-get_account_program_linksrVsK VRUU$4b9 V%56JJsO$!12#%C 5D 3Z)'0@A `AEE&M'R..0D44< HHTN!%%/*F6 JJv!4" ; ; =tW]^ _ `'v/?@  J 4VR(!)$ J=  s E E,+E,cpd} |j|}|xsdjS#t$rYywxYw)z8Best-effort extraction of the 'Program Summary' section.a function norm(s){return (s||'').replace(/\s+/g,' ').trim();} const headers = Array.from(document.querySelectorAll('h1,h2,h3,h4,legend,strong')); let h = headers.find(x => norm(x.innerText).toLowerCase() === 'program summary'); if (!h) { // fallback: contains h = headers.find(x => norm(x.innerText).toLowerCase().includes('program summary')); } if (!h) return ''; // Walk forward through siblings collecting text until the next header-like element. let out = []; let n = h.nextElementSibling; let guard = 0; while (n && guard < 200) { const tag = (n.tagName||'').toLowerCase(); const txt = norm(n.innerText); if (['h1','h2','h3','h4','legend'].includes(tag)) break; if (txt) out.push(txt); n = n.nextElementSibling; guard++; } return out.join('\n\n'); rE)rrNr^)rrtxts r-extract_program_summary_blockrYsCF0##F+ r  "" s $) 55c|Dcgc] }|jj"}}d} |j||Scc}w#t$rYywxYw)zfReturn a table id for the first table that contains all required headers (case-insensitive substring).a6 const required = arguments[0]; const tables = Array.from(document.querySelectorAll('table')); for (const t of tables) { const ths = Array.from(t.querySelectorAll('thead th')).map(th => (th.innerText||'').trim().toLowerCase()); if (!ths.length) continue; let ok = true; for (const r of required) { if (!ths.some(h => h.includes(r))) { ok = false; break; } } if (ok) { return t.id || null; } } return null; N)rNrerr^)rrequired_headersr>requiredrs r-find_table_by_headersr]sZ+; o.value === '-1') || opts.find(o => (o.text||'').toLowerCase().includes('all')) || opts[opts.length-1]; if (!chosen) return false; sel.value = chosen.value; sel.dispatchEvent(new Event('change', {bubbles:true})); return true; N)rr^rs r-rCrC s0 F fh/    s  ##c\d} t|j||S#t$rYywxYw)zAClick Next for a DataTable if available. Returns True if clicked.a const tableId = arguments[0]; const pag = document.getElementById(`${tableId}_paginate`); if (!pag) return false; const next = pag.querySelector('a.paginate_button.next'); if (!next) return false; const cls = (next.className||'').toLowerCase(); if (cls.includes('disabled') || next.getAttribute('aria-disabled') === 'true') return false; next.click(); return true; F)rrr^rs r-rGrGs8 FF))&(;<< s  ++cd} t|j||}|r"tjdt |dyy#t $rYywxYw)z:Best-effort: jump DataTables pagination to the first page.a6 const tableId = arguments[0]; const pag = document.getElementById(`${tableId}_paginate`); if (!pag) return false; const isDisabled = (a) => { if (!a) return true; const cls = (a.className||'').toLowerCase(); if (cls.includes('disabled')) return true; if (a.getAttribute('aria-disabled') === 'true') return true; return false; }; // Prefer "First" button if present const first = pag.querySelector('a.paginate_button.first'); if (first && !isDisabled(first)) { first.click(); return true; } // Otherwise click the smallest numbered page button (usually page 1) const nums = Array.from(pag.querySelectorAll('a.paginate_button')).filter(a => { const cls = (a.className||'').toLowerCase(); if (cls.includes('previous') || cls.includes('next') || cls.includes('first') || cls.includes('last')) return false; const txt = (a.innerText||'').trim(); return /^\d+$/.test(txt); }); if (nums.length > 0) { nums.sort((a,b) => parseInt(a.innerText.trim(),10) - parseInt(b.innerText.trim(),10)); const btn = nums[0]; if (!isDisabled(btn)) { btn.click(); return true; } } return false; g?N)rrrrrr^)rrrrKs r-rDrD1sWF> v,,VX>?  JJt   ,     s>A AAc|xsdjj}|xsdjj}d}g}|D]0}|D])}||z}||kr||kDr|ccS|j|+2|S)zLReturn a list of two-letter search terms from start..end inclusive (aa..zz).rzzabcdefghijklmnopqrstuvwxyz)rNrer()r-rletterstermsrbts r-iter_two_letter_termsri[s ]d ! ! # ) ) +E ;$    % % 'C*GE  AAA5y3w LLO   Lr,cdttd} |jrytj|j dxsd}td|j dxsgD|d<td|j dxsgD|d<|S#t $rY|SwxYw) zLoad scraper resume state from disk. Backward compatible with older state files that only stored `scanned_account_urls`. scanned_account_urlscompleted_termsutf-8encoding{}c32K|]}t|ywrr$rxs r-rz$load_scrape_state..ys/i1A/irlc3jK|]+}t|jj-ywrr$rNrerts r-rz$load_scrape_state..zs#*oa3q6<<>+?+?+A*o13rm)rErZjsonloads read_textrpr^) state_pathstatedatas r-load_scrape_staterms !$5 E    ::j22G2DLMD,//iJ`AaAgeg/i,iE( )'**oDHHUfLgLmkm*o'oE# $ L   L sB B"" B/.B/ci} tj|xsd}td|j dxsgDtd|j dxsgDdS#t$ri}YXwxYw)Nrqc32K|]}t|ywrrsrts r-rz/_load_scrape_state_from_text..s#]qCF#]rvrlc3jK|]+}t|jj-ywrrxrts r-rz/_load_scrape_state_from_text..s#c!s1v||~335cryrmrk)rzr{r^rErp)rrs r-_load_scrape_state_from_textrs| Dzz#+&!$#]TXX>T5U5[Y[#] ]cIZ@[@a_acc  sA(( A65A6cLtd|Dtd|DdS)Nc38K|]}|st|ywrrsrts r-rz!_state_payload..s&D!!s1v&Dsc3pK|].}|st|jj0ywrrx)rrhs r-rz!_state_payload..s&!WQUV#a&,,."6"6"8!Ws6,6rk)sorted)scannedrms r-_state_payloadrs( &&Dw&D D!!W/!WW r,cLt3tj|jtjy|j d t j |jt jdy#t$rtjdYnwxYwZ)Nrr g?) fcntlflockfilenoLOCK_EXseekmsvcrtlockingLK_LOCKOSErrorrrfhs r-_lock_file_exclusiverss  BIIK/GGAJ   NN299; :   JJt   s 3BB"!B"ct3tj|jtjy|j d t j |jt jdy#t$rYywxYw)Nrr ) rrrLOCK_UNrrrLK_UNLCKrrs r- _unlock_filers_  BIIK/GGAJ ryy{FOOQ7    s 3B B  B c0|jjdd|jdd5}t| |j dt |j t|cdddS#t|wxYw#1swYyxYw)NTra+rnror)rrr\rrrr]r)r}rs r-_locked_read_scrape_statersD48  0BR   GGAJ/ :      s" B )A<' B < B  B  Bc |jjdd|jdd5}t| |j dt |j }t|dt|z}t|dt|z}t||}|j d|j|jtj|d d |jtj|j!t#| d d d y #t#|wxYw#1swYy xYw#t$$rYy wxYw) zMPersist resume state, merging with on-disk state (safe for parallel workers).Trrrnrorrlrm)indent sort_keysN)rrr\rrrr]rErtruncatewriterzdumpsflushrifsyncrrr^)r}rrmrcurrentmerged_scannedmerged_completedpayloads r-save_scrape_staters$ t< __TG_ 4 !  $ ! 6rwwyA!$W-C%D!EG !T#&w/@'A#BSEY#Y (9IJ  GAFG %R  ! !R  ! !    sF0E E C%D9$ E 0E9 EE  EEE E! E!cF t|}||dvS#t$rYywxYw)NrlF)rr^)r} account_urlr~s r-is_account_scannedrs4)*5e$:;;; s   c|dz }|jddtj|jddj }||dz S)Nz.voucher_account_claimsTrrnignore)errorsz.lock)rhashlibsha1encode hexdigest)base_dirr claims_dirdigests r-_account_claim_pathrsY55JTD1 \\+,,WX,F G Q Q SF 6(%( ((r,ct||}tj}tdD]} tjt |tj tjztjzd}tj|dd5}|jdtjd|jdt|d|jd |dd d d |cSy #1swYxYw#t$rN ||jjz }||kDr|j!d Y" Yy #t"$rYYy wxYwt"$rYy wxYw) zKBest-effort account claim to reduce duplicate work across parallel workers.riwrnrozpid=rHztime=zurl=NT missing_ok)rrrrir\r$O_CREATO_EXCLO_WRONLYfdopenrgetpidintFileExistsErrorstatst_mtimeunlinkr^) rr stale_seconds claim_pathnowrKfdrages r-try_claim_accountrsQ$X{;J ))+C 1X Z"**ryy*@2;;*NPUVB2sW5 14 }B/05S "-.4 }B/0 1 & ! 1 1   JOO-666&%%%6'     sIA(D AD3 D D D  E/4E E E/E  E/.E/cN|sy |jdy#t$rYywxYw)NTr)rr^)rs r-release_account_claimrs0  T*    s  $$cptjdd|xsdjjS)NrJrGrE)rOrPrNre)rTs r- _norm_namers) 66&#R0 1 7 7 99r,ct|dk7ry|Dchc]\}}t|j}}}|tk(Scc}}w)N F)rrr%SPECIAL_SLOW_RETRY_ACCOUNTS)accountsrrKnamess r-should_retry_term_in_slow_modersB 8}2: ;wtQZ " ;E ; / // rsigrxrKs r-extract_all_datatable_rowsrsDGDVX.JJsO$ 5D 3Z) 2 " D=! $xx 28b:1::((6"(b ,AEE'N(bE((5!9%Cd{ HHSM KKE*q* +  ,'vx8  D= 3VR(%)( D=!;+s 4 D1% D6 cV t|tjtdd} |j |t}|syt |jdxsdj|jdxsdjS#t$rYywxYw#t$rYywxYw)zGIf the first program row is active (checkbox checked), return its link.rNa& const table = document.getElementById(arguments[0]); if (!table) return null; const firstRow = table.querySelector('tbody tr'); if (!firstRow) return null; const cb = firstRow.querySelector('input[type="checkbox"]'); const active = cb ? (cb.checked || cb.getAttribute('checked') !== null) : false; if (!active) return null; const a = firstRow.querySelector('a[href]'); if (!a) return null; return {text: (a.innerText||'').trim(), href: a.getAttribute('href')||''}; r%rEr&r) rrrrTr rr#rprNr^)rrrOs r-get_first_active_program_linkr8sVRUU$4b9 F##F,<=cggfo3::xs|j1d?xsd@}|j1|t5j6||jKtLjNdAjF}tQ|}tS|}|xs|j1dBxs |jF}|j1dCxs|}|j1dDxsd@} |r|dE|n|}!| r|!dE| }!|tU|!dFGz }"|"jWd'd'HtY|d'I}#|#D$cgc]}$t[|$j1dJs|$!}%}$t'dKt)|%|%s;g}&|&j]dL||r|&j]dM|| r|&j]dN| |&j]dO||&j]dP| |&j]dQ|j_D]\}'}(|&j]dR|'dS|(|&j]dT|&j]dU|&j]||&j]dVdWja|&dWz})tc|"dX|)|j=|t?||| te|y|%D]}$|$j1dYxsd@jgxsdZ}*|$j1d[xsd@jg}+tA||+},t'd\|*|j1|,t5j6|ti|}-|-s*|jKtLjNdAjF}-g}.g}/tk|d]d^g}0|0rtm||0\}.}/to|.|/\}.}/n= 1z5--worker-index must be between 0 and --worker-count-1zvoucher_scanned_accounts.jsonrlrm)rrczSearch terms: z .. r z (total )zWorker /z: z assigned terms.zResume (local snapshot): z completed terms, z scanned accounts.zFirst assigned term: z!No terms assigned to this worker.zQ ================================================================================z SEARCH TERM: zP================================================================================Fgffffff?zTDetected known 9-account partial-load pattern. Retrying same term with slower waits.Tg@g333333?rrg@zNo accounts returned for term 'z'.zAccounts found for 'z': zSlow mode active for this term.g?g?rrgffffff?g333333?)r-[z] SKIP (already scanned): z$] SKIP (claimed by another worker): z ] Account: z Account #zAccount#col_2rEbodyr%r&r$rKrR)rUr)rUrSz Active programs: z# z - **Account #**: z- **Parent Account**: z- **Account URL**: z- **Search Term**: z$ ## Account Row (from results table)z- **z**: z ## Account Page Textz```textz```rHzAccount Summaryr%Programr&z - Stylez Product NameProductz - **Account Name**: z- **Account #**: z- **Program URL**: z ## Program Summaryu ## Product List (all rows) — z rowsz```csv,c3@K|]}|jddyw)r;N)rM)rrs r-rzmain..)s1_!!))C2E1_srGrz._No product table found (or it had no rows)._ z All terms completed.):argparseArgumentParserrr parse_argsrerrorrrrrrrrrrrirrrrrrr^rp current_urlrrrLrrFrr r&rr%rrrTAG_NAMEr3r7rVrrVrr(r)rrrrNrYr]rr?rMquit)4apargsrr} scrape_statescanned_accountsrmrrfrrh worker_termsr current_stateterm_slow_modeaccounts_list_urlraccount_open_sleepprogram_open_sleepaccount_return_sleeplist_return_sleepr account_linkrrracct_numaccount_page_text company_info span_namer%r&r$ folder_label out_folderprogramsr`active_programsr,r r2 md_content program_name program_href program_urlprogram_summary_textproduct_headers product_rows prod_table_idr>rxcleaneds4 r-mainr&^s  "BOOL)??QRUVfRgQhhz{|  ),q/):; < 5 6 D MD5jAM  .D E E  }->? ?O& / " M$( ) (O"N 3$VT2!' 2 2 4VP (c#h-A B78(6C (6C *83c '53 09(!0LF 6,,lG%&79J9JK "226HU`6a$((5AcU!CM?2L\M^M^L_`a.xE !AcU!CM?2VWcWhWhVijkz6AcU!CM?+l>O>O=PQR&{{;7p7;;z;RpV]VaVabiVjpnpHJJ{+JJ12(.(;(;BKK(P(U(U%#FGX#YL B6 JI#,#e 0@0@0P#eT`TeTeL%1%5%56F%G%S8N%1%5%56F%G%M2NIWl^1^4D#E`l_mL%*6q8H'I !),>|UX,Y!YJ$$TD$A8TRH2:&TQd155?>Sq&TO&T/O0D/EFG*+- r,%89)!LL+>~>N)OP)!LL+A.AQ)RS ':;-%HI ':4&%AB %LM$+MMO Y/ %67 U+%)YYu%5%< &z3DjQ(,,[9)*6FX P**5M->9()f (;'B'B'D'Q ()f (;'B'B'D &-k<&H |n56 ;/ #56/LV/T,3393F3Fr{{TZ3[3`3`0578: (=fwP^F_(` ( ? @"LL/!LL)Z[%)YYu%5%< &z<L ;/ #78}>9@%((5%j2BOTJJ01JJ01)*5MF 6P    % j*:O LID ML &'  mh4 3JJv112JJsO $VT2  3&!s #45 3$(s]`nqrr sz'UZ+f,**5  s)n3$k*k.C+n3 k%&/n3m&En3.En# n,n0E n#: n3I0n#7-n $D/n#:n3n3% l?/0l l? l,)l?+l,,l?;n3>l??n3 n &m32n3 m?<n>m??nn3nn3 n## n00n33o__main__))rTr$rUrrr$)r_r$rr)r Optional[str])rr)rr))rr)rr)rNone)rrrzwebdriver.Firefox)r)rr)rrrr*)rr$rzDict[str, object]) rr$rfloatrr+rr+rr*)rzTuple[LinkCell, Dict[str, str]])r*r$rzDict[str, str])rr$)r List[str]rzList[List[str]]r!Tuple[List[str], List[List[str]]])r%List[Tuple[LinkCell, Dict[str, str]]])rr$rList[Dict[str, object]])F)rUrrr/)r[z Sequence[str]rr))rr$rr*)rr$rr)rrc)r-r$rr$rr,)r}rrDict[str, set])rr$rr0)rrErmrErzDict[str, List[str]])r}rrrErmrErr*)r}rrr$rr)rrrr$rr)i`T)rrrr$rrrOptional[Path])rr1rr*)rTr$rr$)rr.rr) rr$rrrrrr,rr)rr$rr-)rzOptional[LinkCell])rrrr$rr$rr)rr)h__doc__ __future__rrrrzrirOrsrr ImportErrorr dataclassesrpathlibrtypingrrrr r urllib.parser seleniumr selenium.common.exceptionsr selenium.webdriver.common.byrselenium.webdriver.common.keysr'selenium.webdriver.common.action_chainsr"selenium.webdriver.firefox.optionsr"selenium.webdriver.firefox.servicerselenium.webdriver.supportrrselenium.webdriver.support.uirrrrrrrrTONLY_ACTIVE_PROGRAMSrr#rSrVrbrfrkr{rrrrrrrrrrrrr3r7r?rLrPrVrYr]rCrGrDrirrrrrrrrrrrrrrrrrr&r' SystemExitr+r,r-rDs&:# "88 7+/@66@7 U   !&$  $ 8 $ 5p9x*$  !F]Y 3"6H!! 5 5 5  5  5 5p5 cD;/~ !<(V*%PB2 ($' T$&    .)6 :00D8l^ z TV y( EsF F! F!