bi(\dZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z n#e $r dZ ddl Z YnwxYwddl mZddlmZddlmZmZmZmZmZddlmZddlmZdd lmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z)ddl*m+Z+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4hdZ5edGddZ6hdZ7ddd%Z8dd(Z9dd)Z:dd*Z;dd,Zdd2Z?dd5Z@ddd8ZAddd9ZBddd:ZCdd=ZDdd>ZEd?d@dAdBddHZFddIZGddKZHddNZIddOZJddUZKddWZLddYZMddd\ZNdd]ZOdd`ZPddaZQddbZRddcZSdddgZTddkZUddmZVddrZWddsZXddtZYdduZZddvZ[ddxZ\ddzZ]ddd~Z^ddZ_ddZ`ddZaddZbddZcddZdddZeddZfegdkrehefdS)aVoucher_List_Scraper.py USAGE pip install selenium # Ensure Firefox + geckodriver are installed. # If geckodriver is not on PATH, set: # export GECKODRIVER_PATH=/full/path/to/geckodriver python3 Voucher_List_Scraper.py python3 Voucher_List_Scraper.py --headless WHAT THIS DOES - Opens the Accounts list page and logs in (credentials hard-coded below). - Clicks id=accSearchBox, types "aa", waits 3 seconds, then presses Enter. - Waits a moment for the results to refresh. - Clicks the FIRST account row in the results table id='device-grid-list'. - Creates a folder in the same directory as this .py. - Writes a Markdown file inside that folder named after the clicked account text. - On the account page, finds id=ProgramListView; if the FIRST program row is active (checkbox checked), it records that program name and opens it. - Records "Program Summary" section (best effort) and captures all rows from the main DataTables product list (not just the first 10 displayed). NOTES - The portal uses DataTables; to avoid stale element errors, this script snapshots tables with JavaScript (reads text + hrefs from the DOM) instead of reading WebElement.text repeatedly. ) annotationsN) dataclass)Path)DictListOptionalSequenceTuple)urljoin) webdriver)TimeoutException)By)Keys) ActionChains)Options)Service)expected_conditions) WebDriverWaitzFhttps://portal.redwingforbusiness.com/RWS_AccountsListPage?tab=accountzwalpine614@rwfb.comzSammiandCoco2!aa accSearchBoxzdevice-grid-listProgramListViewALLT> town of la platatown of indian headbolling afb commissarycharles county utilitiesbattle creek constructioncharles county public works!charles county emergency services!maryland transportation authority-smithsonian - african american history museum)frozenc$eZdZUded<ded<dS)LinkCellstrtexthrefN)__name__ __module__ __qualname____annotations__voucher_scraper_core.pyr$r$_s" III IIIIIr-r$>AUXCONNULPRNCOM1COM2COM3COM4COM5COM6COM7COM8COM9LPT1LPT2LPT3LPT4LPT5LPT6LPT7LPT8LPT9sr%max_lenintreturnc|pddddddd}tjdd|}tjdd|}tjd d|}|d }|sd }|t vr|d}|d |S) zReturn a filesystem-safe single path component. - Preserves spaces and '$' (so you can get names like '$150 Subsidy Safety Toe.md'). - Removes characters that are invalid on Windows and path separators on all OSes.     \s+z [<>:"/\\|?*]_z [\x00-\x1f]z .itemN)replacestripresubrstripupper_WINDOWS_RESERVED)rFrGs r._safe_fs_componentrZls b$$$,,T377??cJJPPRRA vsAA Q''A ~r1%%A A  wwyy%%%GGG XgX;r-pathboolc( t|}|r|sdS|d5}|ddkcdddS#1swxYwYdS#t $rYdSwxYw)zNReturn True if `path` looks like a real ELF binary (not a shell/snap wrapper).FrbsELFN)rexistsis_fileopenread Exception)r[pfs r._is_elf_executablergs JJxxzz  5 VVD\\ +Q66!99 * + + + + + + + + + + + + + + + + + + uus:7BBA6) B6A::B=A:>B BBc t|}|o0|o|jdkS#t $rYdSwxYw)Nz.exeF)rr`rasuffixlowerrd)r[res r._is_windows_executablerkse JJxxzzHaiikkHahnn.>.>&.HH uusAA A$#A$c^tjdkrt|St|S)Nnt)osnamerkrg)r[s r._is_usable_executablerps) w$%d+++ d # ##r- Optional[str]ctjd}|rtt |}t |r|Stt |d}t|r|Stj dkrFddtj dtj dg}|D]}|rt|r|cSdSd d g}|D]}t|r|cStj dtj dtj d g}|D]s}|rt|r|cS|rZt |j dkrBtt |d}t|r|cStdS) zFind a real Firefox executable for GeckoDriver. On many Pi / Ubuntu setups, `firefox` on PATH is a snap wrapper script, which GeckoDriver rejects. We prefer `firefox-bin` (real ELF) when available. FIREFOX_BINARYz firefox-binrmz,C:\Program Files\Mozilla Firefox\firefox.exez2C:\Program Files (x86)\Mozilla Firefox\firefox.exez firefox.exefirefoxNz1/snap/firefox/current/usr/lib/firefox/firefox-binz-/snap/firefox/current/usr/lib/firefox/firefoxz firefox-esr) rnenvirongetr%r expanduserrp with_namergroshutilwhichrk)env_binsibwin_candidatescsnap_candidatespath_candidatess r._locate_firefox_binaryrsjnn-..Gd7mm..0011  ) ) N$w--))-8899 c " " J w$ ; A L ' ' L # #      A +A.. t <7O a  HHH   ]## Y ]##O   #A&& HHH  a**d1gg'' 6677C!#&&  4r- firefox_binctjd}|r?tt |}t |r|Stjdkr|r@tt |d}t|r|Stj dtj dg}|D]}|rt|r|cStt t jdz ddg}|D]}t|r|cSdSd}t|r|S|r@tt |d}t|r|Stj d}|rt|r|Sgd }|D]}t|r|cSdS) z>))++25FFGGH E 4U44555 E5XFFF F E E E58DDD D D D E Z[[[*(++++ ***))))))*s$ A11BB"B22CCNonecr ttj}|dz }|ddt jdt|t jdt|dz dS#t$rYdSwxYw)zIUse a project-local Selenium cache to avoid unwritable default locations.z.selenium-cacheTparentsexist_ok SE_CACHE_PATHXDG_CACHE_HOMEz.cacheN) rrrrmkdirrnru setdefaultr%rd)base cache_dirs r._configure_selenium_runtime_envr)s H~~%%''.,, t444 os9~~>>> .D8O0D0DEEEEE      sB$B(( B65B6headlesswebdriver.Firefoxctt}|r|dt}|rt d|||_nt d|dd|dd tjt||}n#t$r}tj d krkt d t d |rd nd t d|pdt dt|rd nd t dwxYw |ddn#t$rYnwxYw|S)N --headlessz[driver] Using firefox binary: zf[driver] WARNING: Could not locate a real Firefox binary; GeckoDriver may fail on snap-wrapper setups.zdom.webnotifications.enabledFzmedia.volume_scalez0.0)serviceoptionsrmz[driver] Windows setup check:z[driver] Firefox detected: yesnoz[driver] Firefox path: z (not found)z![driver] geckodriver detected: zT[driver] Tip: install geckodriver.exe and put it on PATH, or set GECKODRIVER_PATH.ii)rr add_argumentrrbinary_locationset_preferencer Firefoxrrdrnrorset_window_size)roptsrdrivers r. build_driverr5s#%%% 99D( ,''')**Kx = ==>>>* vwww 6>>>,e444 "+? +L+LVZ[[[  7d?? 1 2 2 2 R;2P%%DRR S S S Lk.J]LL M M M K1+>>HUUDKK L L L h i i i  tS))))      Ms#B>>BE E E-,E-timeoutcpt||tj||fSN)runtilECpresence_of_element_locatedrbyvaluers r. wait_presentrXs1  ) ) / /0NPRTY{0[0[ \ \\r-cpt||tj||fSr)rrrelement_to_be_clickablers r.wait_clickabler\s0  ) ) / /0JBPU;0W0W X XXr-c^tj|z}tj|kr |tjd}|sdSt d|DsdSn#t $rYdSwxYwtjdtj|k}dSdS)z@Best-effort wait for DataTables processing overlay to disappear.zdiv.dataTables_processingNc3>K|]}|VdSr) is_displayed.0res r. z'datatables_wait_idle..hs,77Aq~~''777777r-g?)time find_elementsr CSS_SELECTORanyrdsleep)rrendprocss r.datatables_wait_idler`s )++ C )++   ((:UVVE 7777777      FF  3 )++      s"A/A// A=<A=table_idDict[str, object]c2d}|||S)zPReturn headers + row cell texts + first link (text+href) per row for a table id.as const table = document.getElementById(arguments[0]); if (!table) return null; const headers = Array.from(table.querySelectorAll('thead th')).map(th => (th.innerText || '').trim()); const rows = Array.from(table.querySelectorAll('tbody tr')).map(tr => { const cells = Array.from(tr.querySelectorAll('td')).map(td => (td.innerText || '').trim()); const a = tr.querySelector('a[href]'); const link = a ? {text: (a.innerText||'').trim(), href: a.getAttribute('href')||''} : null; return {cells, link}; }); return {headers, rows}; )execute_scriptrrscripts r.js_table_snapshotros! F   2 22r-c|tt|tjdd}t|tjdd} |d|n#t $rYnwxYw |n#t $rYnwxYw|t |n#t $rYnwxYw|t | tjd n/#t $r"|tjYnwxYwt|tjtddS)Nusernamerpasswordz.arguments[0].scrollIntoView({block:'center'});Login<)rv LOGIN_URLrrIDrrdclear send_keysUSERNAMEPASSWORD find_elementclickrENTERACC_SEARCHBOX_ID)ruser_elpass_els r.loginrs JJy625*b99G625*b99G NPWXXXX               h         h&BE7++113333 &&&$*%%%%%& 0"55555sHA++ A87A8<B BB<C CC<2D//)EE@??pre_enter_sleeppost_enter_sleep final_sleeptermrfloatrrc|pd}td|dt|tjt d}| |na#t$rT | tj d| tj n#t$rYnwxYwYnwxYw| |tj| |n#t$rYnwxYw | tjn#t$rYnwxYw | tjn#t$rYnwxYw t%| tjn#t$rYnwxYw | tjn#t$rYnwxYwtj|t+|dtj|dS)z6Type `term` into accSearchBox, wait, then press Enter.rKzFiltering accounts: typing 'z' into accSearchBox...raN)rTrjrrrrrrrrdrrCONTROL BACKSPACErrrRETURNrperformTABr)rrrrrboxs r.filter_accounts_termrs| JB     % % ' 'D E E E EFFF (8" = =CIIKKK   MM$, , , , MM$. ) ) ) )    D  MM$J         dj!!!!        dk""""       V&&tz22::<<<<        dh       J   $$$J{s2B C%?CC% CC%CC%$C%D'' D43D48E E%$E%)F FF>G G&%G&*H HHc,t|tS)z&Back-compat wrapper: uses SEARCH_TERM.)r SEARCH_TERM)rs r.filter_accounts_simplers  4 44r-Tuple[LinkCell, Dict[str, str]]c<t|tjtddd}t |d|t |t}|r|dstdd|dpgD}|dd }|d }|r|d std |d pg}i}t|D](\}} |t|kr|||| pd|<)t|dd |d d |fS)zHReturn the first account link + row mapping from accounts results table.rrIr\c |tjt}|tjd}t |dkS#t$rYdSwxYw)Nztbody tr td a[href]rF)rrrACCOUNTS_TABLE_IDrrlenrd)dtblrs r._has_first_linkz7get_first_account_from_results.._has_first_linksg ..(9::C!!"/3HIIAq66A:    55 sAA A('A(rowsz)Accounts results table snapshot is empty.cg|]}|Sr,r,rhs r. z2get_first_account_from_results..sAAA!AAAr-headersrlinkr'z9Could not find a clickable account link in the first row.cellscol_r&rKr&r')rIr\) rrrrrrrrv RuntimeError enumeraterr$rT) rrsnapr firstr r row_mapirs r.get_first_account_from_resultsrs 12666&"##O444 V%6 7 7D Htxx''HFGGGAAdhhy&9&9&?RAAAG LOE 99V  D Xtxx''XVWWWyy))/RE G'""001 s5zz>>',QxGAO $ &"--3355DHHVR}|||vrn1 ||t dkrn?d fd }i} |d | d<|d| d<|dp |d| d<d| DS)zParse Company Information from the Account Summary page text. Parent Account is the first line under 'Company Information' (if present). We pull: - parent_account (optional) - company_name - account_number c6g|]}|Sr,)rTrlns r.rz7parse_company_information_from_text..s A A ABRXXZZ A A Ar-rKc6g|]}|Sr,)rXrs r.rz7parse_company_information_from_text.. s ( ( (BRXXZZ ( ( (r-zCOMPANY INFORMATION>COMPANY ADDRESSTAX INFORMATIONCONTACT INFORMATIONCREDIT & BILLING INFORMATION#PAYMENT TERMS & INVOICE PREFERENCESACCOUNTSPROGRAMS ATTACHMENTSPlabelr%rIc|}tD]\}}|}||kr\t|dztD];}|}|s|dvr6|ccS||dzr+|t|dcSdS)Nr> ACCOUNT # COMPANY NAMEACCOUNT NUMBERPARENT ACCOUNTrMrK)rXrrangerrT startswith)r'lab_uidxrawruknxtsections r. _value_afterz9parse_company_information_from_text.._value_after&s !'** 0 0HCBU{{sQwG 55A!!***,,C! yy{{&ggg JJJJJ}}US[)) 03u::;;'--///// 0rr-r,parent_accountr* company_namer+r)account_numberci|] \}}||| Sr,r,)rr3vs r. z7parse_company_information_from_text..:s# . . .TQA .Aq . . .r-)r'r%rIr%) splitlinesrr-rappenditems) rlinesrXstartru stop_headersjr6outr5s @r.#parse_company_information_from_textrFs B A9?">">"@"@ A A AE ( (% ( ( (E E%  1 % % %)>!)C)CEE E*D {{    LG 5#e** % % 8| # # EuQx   w<<2   E  C(L)9::C&,~66C(L)9::Wll;>W>WC . .SYY[[ . . ..r-c t|tjdd|tjd}|jpdS#t $rYdSwxYw)zGet the account name from the Account Summary header span. After clicking an account row, the portal shows the canonical account name in: Aaron Equipment Company z AccountSummary:accountForm:aname rK)rrrrr&rTrd)rels r."get_account_name_from_summary_spanrJ>sp VRU$FKKK  (J K K 2$$&&& rrsAA A'&A'r  List[str]rList[List[str]]!Tuple[List[str], List[List[str]]]cN ddh}g}tpgD]J\}}|pd}|s||vr5||K|s|fSfd|D}g}|pgD]# | fd|D$||fS)aDrop backend/hidden columns that DataTables sometimes includes in the DOM. The Red Wing portal product table often contains extra non-user columns such as: - Product Id - Filter Number These may be invisible in the UI but appear in our scrape output. z product idz filter numberrKc g|] }| Sr,r,)rrr s r.rz/drop_hidden_product_columns..cs000!71:000r-cJg|]}|tkr|nd S)rK)r)rrrs r.rz/drop_hidden_product_columns..fs/HHH!!c!ff**!A$$"HHHr-)rrTrjr>) r r drop_nameskeep_idxrrhh new_headersnew_rowsrQs ` @r.drop_hidden_product_columnsrWLs0JH'-R((1g2__     88:: # #  }0000x000K "H ZRJJHHHHxHHHIIII   r-%List[Tuple[LinkCell, Dict[str, str]]]crt|tjtdt |tt jdt|dt|tg}t}tdD]}t|t}|r| dsnvd| dpgD}| dpgD]}| dpi}| d pd }|r||vrI||t| d pd | } | d pg} i} t!|D](\} } | t#| kr| | | | pd| <)|| | ft'|t}|sn&t jdt|d|S)zHReturn ALL account links + row mappings from the accounts results table.r皙?ri,rcg|]}|Sr,r,rs r.rz1get_all_accounts_from_results..|sEEEAaEEEr-r r r'rKr&r r r ffffff?)rrrrset_datatable_length_allrrrdatatable_goto_firstsetr-rrvrTaddr$rrr>datatable_click_next)rrEseenrQrr rowr r' link_cellr rrrclickeds r.get_all_accounts_from_resultsrfjs5 12666V%6777JsOOO$$$!233313C 55D 3ZZ)) ):;; 488F++  EEE$((9*=*=*CEEE88F##)r - -C776??(bDHHV$$*1133D 44<< HHTNNN txx'7'7'=2&D&D&F&FTRRRI"www//52E&(G!'** 8 81s5zz>>/4QxGAO, JJ 7+ , , , ,&v/@AA  E 4VR(((( Jr-List[Dict[str, object]]czd} |||}t|pgS#t$rgcYSwxYw)z>Snapshot program rows: link text/href + checkbox active state.aF const table = document.getElementById(arguments[0]); if (!table) return []; const rows = Array.from(table.querySelectorAll('tbody tr')).map(tr => { const a = tr.querySelector('a[href]'); if (!a) return null; const cb = tr.querySelector('input[type="checkbox"]'); const active = cb ? (cb.checked || cb.getAttribute('checked') !== null) : false; return {text: (a.innerText||'').trim(), href: a.getAttribute('href')||'', active: active}; }).filter(x => x && x.href); return rows; )rlistrd)rrrress r.js_program_list_snapshotrksW F##FH55CI2  s &+ ::F only_activecZ t|tjtdn#t$rgcYSwxYwt |tt jdt|dt|tg}t}tdD]}t|t}|D]}| dpd}|r||vr2||t!| d}|r|sn|| dpd||d t%|t} | sn&t jd t|d|S) zuReturn program links from the 'List Of Account Related Programs' table. Returns dicts: {text, href, active} rrZrr'rKactiver&)r&r'ror\)rrrPROGRAM_TABLE_IDr r]rrrr^r_r-rkrvrTr`r\r>ra) rrlrErbrQrrQr'rores r.get_account_program_linksrqs VRU$4b9999  V%5666JsOOO$$$!1222#%C 55D 3ZZ))'0@AA ` `AEE&MM'R..00D 44<< HHTNNN!%%//**F 6  JJv!4" ; ; = =tW]^^ _ _ _ _&v/?@@  E 4VR(((( Js !$ 33cd} ||}|pdS#t$rYdSwxYw)z8Best-effort extraction of the 'Program Summary' section.a function norm(s){return (s||'').replace(/\s+/g,' ').trim();} const headers = Array.from(document.querySelectorAll('h1,h2,h3,h4,legend,strong')); let h = headers.find(x => norm(x.innerText).toLowerCase() === 'program summary'); if (!h) { // fallback: contains h = headers.find(x => norm(x.innerText).toLowerCase().includes('program summary')); } if (!h) return ''; // Walk forward through siblings collecting text until the next header-like element. let out = []; let n = h.nextElementSibling; let guard = 0; while (n && guard < 200) { const tag = (n.tagName||'').toLowerCase(); const txt = norm(n.innerText); if (['h1','h2','h3','h4','legend'].includes(tag)) break; if (txt) out.push(txt); n = n.nextElementSibling; guard++; } return out.join('\n\n'); rK)rrTrd)rrtxts r.extract_program_summary_blockrtsYF0##F++ r  """ rrs */ ==required_headers Sequence[str]cnd|D}d} |||S#t$rYdSwxYw)zfReturn a table id for the first table that contains all required headers (case-insensitive substring).cZg|](}|)Sr,)rTrj)rrQs r.rz)find_table_by_headers..s*<< (th.innerText||'').trim().toLowerCase()); if (!ths.length) continue; let ok = true; for (const r of required) { if (!ths.some(h => h.includes(r))) { ok = false; break; } } if (ok) { return t.id || null; } } return null; Nrrd)rrurequiredrs r.find_table_by_headersr{sZ<<+;<< o.value === '-1') || opts.find(o => (o.text||'').toLowerCase().includes('all')) || opts[opts.length-1]; if (!chosen) return false; sel.value = chosen.value; sel.dispatchEvent(new Event('change', {bubbles:true})); return true; Nryrs r.r]r] sL F fh/////      s  **cpd} t|||S#t$rYdSwxYw)zAClick Next for a DataTable if available. Returns True if clicked.a const tableId = arguments[0]; const pag = document.getElementById(`${tableId}_paginate`); if (!pag) return false; const next = pag.querySelector('a.paginate_button.next'); if (!next) return false; const cls = (next.className||'').toLowerCase(); if (cls.includes('disabled') || next.getAttribute('aria-disabled') === 'true') return false; next.click(); return true; F)r\rrdrs r.rara sN FF))&(;;<<< uus "' 55cd} t|||}|r&tjdt |ddSdS#t $rYdSwxYw)z:Best-effort: jump DataTables pagination to the first page.a6 const tableId = arguments[0]; const pag = document.getElementById(`${tableId}_paginate`); if (!pag) return false; const isDisabled = (a) => { if (!a) return true; const cls = (a.className||'').toLowerCase(); if (cls.includes('disabled')) return true; if (a.getAttribute('aria-disabled') === 'true') return true; return false; }; // Prefer "First" button if present const first = pag.querySelector('a.paginate_button.first'); if (first && !isDisabled(first)) { first.click(); return true; } // Otherwise click the smallest numbered page button (usually page 1) const nums = Array.from(pag.querySelectorAll('a.paginate_button')).filter(a => { const cls = (a.className||'').toLowerCase(); if (cls.includes('previous') || cls.includes('next') || cls.includes('first') || cls.includes('last')) return false; const txt = (a.innerText||'').trim(); return /^\d+$/.test(txt); }); if (nums.length > 0) { nums.sort((a,b) => parseInt(a.innerText.trim(),10) - parseInt(b.innerText.trim(),10)); const btn = nums[0]; if (!isDisabled(btn)) { btn.click(); return true; } } return false; g?N)r\rrrrrd)rrrres r.r^r^2sF> v,,VX>>??  - Jt     , , , , , - -      sA A AAzzrArc|pd}|pd}d}g}|D]4}|D]/}||z}||kr||kr|ccS||05|S)zLReturn a list of two-letter search terms from start..end inclusive (aa..zz).rrabcdefghijklmnopqrstuvwxyz)rTrjr>)rArletterstermsrbts r.iter_two_letter_termsr\s ]d ! ! # # ) ) + +E ;$     % % ' 'C*GE   AAA5yy3ww LLOOOO   Lr- state_pathrDict[str, set]cttd} |rtj|dpd}td|dpgD|d<td|dpgD|d<n#t $rYnwxYw|S) zLoad scraper resume state from disk. Backward compatible with older state files that only stored `scanned_account_urls`. scanned_account_urlscompleted_termsutf-8encoding{}c34K|]}t|VdSrr%rxs r.rz$load_scrape_state..zs(/i/i1A/i/i/i/i/i/ir-rc3|K|]7}t|V8dSrr%rTrjrs r.rz$load_scrape_state..{s<*o*oa3q66<<>>+?+?+A+A*o*o*o*o*o*or-r)r_r`jsonloads read_textrvrd)rstatedatas r.load_scrape_staterns !$55  E      p:j22G2DDLMMD,//i/iJ`AaAaAgeg/i/i/i,i,iE( )'**o*oDHHUfLgLgLmkm*o*o*o'o'oE# $      LsB C C  C r1ci} tj|pd}n#t$ri}YnwxYwtd|dpgDtd|dpgDdS)Nrc34K|]}t|VdSrrrs r.rz/_load_scrape_state_from_text..s(#]#]qCFF#]#]#]#]#]#]r-rc3|K|]7}t|V8dSrrrs r.rz/_load_scrape_state_from_text..s<cc!s1vv||~~3355ccccccr-rr)rrrdr_rv)r1rs r._load_scrape_state_from_textrs Dz#+&& !$#]#]TXX>T5U5U5[Y[#]#]#] ] ]ccIZ@[@[@a_accccc  s  **scannedr_rDict[str, List[str]]chtd|Dtd|DdS)Nc38K|]}|t|VdSrrrs r.rz!_state_payload..s-&D&D!!&Ds1vv&D&D&D&D&D&Dr-c3K|]9}|t|V:dSrr)rrs r.rz!_state_payload..sB!W!WQUV!W#a&&,,.."6"6"8"8!W!W!W!W!W!Wr-r)sorted)rrs r._state_payloadrsE &&D&Dw&D&D&D D D!!W!W/!W!W!WWW  r-cVt3tj|tjdS|d t j|t jddS#t$rtj dYnwxYwZ)NrTrg?) fcntlflockfilenoLOCK_EXseekmsvcrtlockingLK_LOCKOSErrorrrfhs r._lock_file_exclusivers  BIIKK///GGAJJJ  N299;; : : : F    Jt       s2BB'&B'c,t3tj|tjdS|d t j|t jddS#t$rYdSwxYw)Nrr) rrrLOCK_UNrrrLK_UNLCKrrs r. _unlock_filers  BIIKK///GGAJJJ ryy{{FOQ77777      s2B BBcn|jdd|dd5}t| |dt |t|cdddS#t|wxYw#1swxYwYdS)NTra+rrr)rrrbrrrrcr)rrs r._locked_read_scrape_statersD4888  0 0BR     GGAJJJ/ ::           s)B*5B:B*B''B**B.1B.cf |jdd|dd5}t| |dt |}t|dt|z}t|dt|z}t||}|d| | tj |d d | tj|t#|n#t#|wxYw d d d d S#1swxYwYd S#t$$rYd SwxYw) zMPersist resume state, merging with on-disk state (safe for parallel workers).Trrrrrrr)indent sort_keysN)rrrbrrrrcr_rtruncatewriterdumpsflushrnfsyncrrrd)rrrrcurrentmerged_scannedmerged_completedpayloads r.save_scrape_staters t<<< __TG_ 4 4 !  $ $ $ ! 6rwwyyAA!$W-C%D!E!EG !T#&w/@'A#B#BSEYEY#Y (9IJJ  GAFFFGGG %%%R    R      ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !      sM3F"FDE3#F3FF F"FF"FF"" F0/F0 account_urlcX t|}||dvS#t$rYdSwxYw)NrF)rrd)rrrs r.is_account_scannedrsG)*55e$:;;; uus  ))base_dirc|dz }|ddtj|dd}||dz S)Nz.voucher_account_claimsTrrignore)errorsz.lock)rhashlibsha1encode hexdigest)rr claims_dirdigests r._account_claim_pathrsh55JTD111 \+,,WX,FF G G Q Q S SF 6((( ((r-`T stale_secondsOptional[Path]ct||}tj}tdD]Q} tjt |tjtjztjzd}tj |dd5}| dtj d| dt|d| d |dd d d n #1swxYwY|cS#t$rR ||jz }||kr|d Y-n#t"$rYnwxYwYd St"$rYd SwxYwd S) zKBest-effort account claim to reduce duplicate work across parallel workers.riwrrzpid=rNztime=zurl=NT missing_ok)rrr-rnrbr%O_CREATO_EXCLO_WRONLYfdopenrgetpidrHFileExistsErrorstatst_mtimeunlinkrd) rrr claim_pathnowrQfdrages r.try_claim_accountrs$X{;;J )++C 1XX Z"*ry*@2;*NPUVVB2sW555 1/ ///000-S---.../ ///000 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1       JOO--66&&%%%666H'    444   444  4saADA*D ? D D DD D F%8E! F! E.+F-E..F5 FFrc^|sdS |ddS#t$rYdSwxYw)NTr)rrd)rs r.release_account_claimrsS  T*****      s  ,,cztjdd|pdS)NrPrMrK)rUrVrTrj)rFs r. _norm_namers0 6&#R00 1 1 7 7 9 99r-accountscZt|dkrdSd|D}|tkS)N Fc<h|]\}}t|jSr,)rr&)rr rQs r. z1should_retry_term_in_slow_mode..s& ; ; ;wtQZ " " ; ; ;r-)rSPECIAL_SLOW_RETRY_ACCOUNTS)rnamess r.should_retry_term_in_slow_moders7 8}}u ; ;( ; ; ;E / //r- worker_count worker_index all_termsct|dkrdS ||}n#t$rYdSwxYw||z|kS)NrTF)index ValueError)rrrrposs r.term_assigned_to_workerr s\qtood## uu , < //s  ..cg}g}t||tjdt|dt }t dD]}t ||}|snd|dpgD}|dpgD]o}|dpg}d|d d } | |vr;| | | d |Dpt||} | sn%tjd t|d||fS) zSExtract all rows from a DataTables table by snapshotting each page and paging Next.rricg|]}|Sr,r,rs r.rz.extract_all_datatable_rows..&s:::1:::r-r rr |Nr_cg|]}|Sr,r,rr~s r.rz.extract_all_datatable_rows..-s***q***r-rZ) r]rrrr_r-rrvjoinr`r>ra) rrr rrbrQrrQr sigres r.extract_all_datatable_rowsr smGDVX...JsOOO$$$ 55D 3ZZ)) 22  E::txx 228b:::((6""(b , ,AEE'NN(bE((5!9%%Cd{{ HHSMMM KK**E*** + + + +'vx88  E 3VR(((( D=r-Optional[LinkCell]c t|tjtdn#t$rYdSwxYwd} ||t}|sdSt |dpd|dpdS#t$rYdSwxYw)zGIf the first program row is active (checkbox checked), return its link.rNa& const table = document.getElementById(arguments[0]); if (!table) return null; const firstRow = table.querySelector('tbody tr'); if (!firstRow) return null; const cb = firstRow.querySelector('input[type="checkbox"]'); const active = cb ? (cb.checked || cb.getAttribute('checked') !== null) : false; if (!active) return null; const a = firstRow.querySelector('a[href]'); if (!a) return null; return {text: (a.innerText||'').trim(), href: a.getAttribute('href')||''}; r&rKr'r ) rrrrpr rr$rvrTrd)rrrjs r.get_first_active_program_linkr 9sVRU$4b9999 tt F##F,<== 4cggfoo3::<p+|d?p|d@pdA}||t7j||&tNj(dBj$}tS|}tU|}|p|dCp|j$}|dDp|}|dEpdA}|r|dF|n|}|r|dF|}|tW|dGHz } | ,d(d(It[|d(J}!dK|!D}"t)dLt+|"|"s}g}#|#.dM||r|#.dN||r|#.dO||#.dP||#.dQ| |#.dR|/D] \}$}%|#.dS|$dT|%!|#.dU|#.dV|#.||#.dWdX0|#dXz}&tc| dY|&||tA||| te||"D] }'|'dZpdA3pd[}(|'d\pdA3})tC||)}*t)d]|(||*t7j|ti|}+|+s%|&tNj(dBj$}+g},g}-tk|d^d_g}.|.r'tm||.\},}-to|,|-\},}-nJtk|d^gptk|d`g}.|.r&tm||.\},}-to|,|-\},}-g}#|#.dM|(|#.da||r|#.db||r|#.dO||#.dP||#.dc|*|#.dQ| |#.dd|#.dV|#.|+|#.dW|#.det+|-df|,r|-r|#.dg|#.dh0di|,D|-D]}/dj|/D}0t+|0t+|,kr&|0dAgt+|,t+|0z zz }0t+|0t+|,kr|0dt+|,}0|#.dh0|0|#.dWn|#.dkdX0|#dXz}&tc| |(|&||t7j| ||tA||||| t7j|te|<#te|wxYw|| tA||| t)dl |8d S#|8wxYw)mNr store_truezRun Firefox headless)actionhelpz--worker-countrz)Total number of parallel workers/browsers)typedefaultrz--worker-indexrzThis worker index (0-based)z--worker-count must be >= 1z5--worker-index must be between 0 and --worker-count-1zvoucher_scanned_accounts.jsonrr)rrc@g|]\}}|jzjk|Sr,)rr)rrrargss r.rzmain..vs3gggda1t?P;PUYUf:f:f:f:f:fr-zSearch terms: z .. rz (total )zWorker /z: z assigned terms.zResume (local snapshot): z completed terms, z scanned accounts.zFirst assigned term: z!No terms assigned to this worker.zQ ================================================================================z SEARCH TERM: zP================================================================================Fgffffff?zTDetected known 9-account partial-load pattern. Retrying same term with slower waits.Tg@g333333?rrg@zNo accounts returned for term 'z'.zAccounts found for 'z': zSlow mode active for this term.g?g?rrgffffff?g333333?)rA[z] SKIP (already scanned): z$] SKIP (claimed by another worker): z ] Account: z Account #zAccount#col_2rKbodyr8r9r7rQrn)rGr)rlcVg|]&}t|d$|'S)ro)r\rvrs r.rzmain..s/&T&T&TQd155??>S>S&Tq&T&T&Tr-z Active programs: z# z - **Account #**: z- **Parent Account**: z- **Account URL**: z- **Search Term**: z$ ## Account Row (from results table)z- **z**: z ## Account Page Textz```textz```rNzAccount Summaryr&Programr'z - Stylez Product NameProductz - **Account Name**: z- **Account #**: z- **Program URL**: z ## Program Summaryu ## Product List (all rows) — z rowsz```csv,c3BK|]}|ddVdS)r&;NrSrs r.rzmain..+s01_1_!!))C2E2E1_1_1_1_1_1_r-cfg|].}|pddddd/S)rKrNrMr&r(r)rs r.rzmain..-s?*e*e*e^_AG+<+r?rrrrTrtr{r rWquit)2apr vouchers_rootr scrape_statescanned_accountsrrr worker_termsr current_stateterm_slow_modeaccounts_list_urlraccount_open_sleepprogram_open_sleepaccount_return_sleeplist_return_sleepr0 account_linkrrracct_numaccount_page_text company_info span_namer8r9r7 folder_label out_folderprogramsactive_programsr@r3r; md_contentre program_name program_href program_urlprogram_summary_textproduct_headers product_rows prod_table_idrQcleanedrs2 @r.mainrQ_s  " "BOOL*>{{RUVfRgRg{{{|||  7 ;,q/;; < < < < 5 6 6 6 D MD MD5jAAM  .D E E  }->? ?O&& / " " " ($(( ) ) ) (OOO"N 3$VT2222 3 3 3JJv1222JsOOOO D$VT22222  3!' 2 4V<PPP AAA#h--AA B B B 97888(6!?C (6!?C *8#A33c '5 >3 09(!0L0L0LF 6F 6,,lG%&79JKK "2226HU`6a6a2$((555`c``CMM``\M^``aaa.xEE !jcjjCMMjjWcWhjjkkkz6QcQQCMMQQl>OQQRRR&{{;77p7;;z;R;RpV]VaVabiVjVjpnpHJJ{+++J1222(.(;(;BK(P(P(U%#FGX#Y#YL B6 J JI#,#e 0@0@0P0P#eT`TeL%1%5%56F%G%G%S8N%1%5%56F%G%G%M2NIW#nl#E#E^#E#E#E`l]nL%J*6'I'I'I'I !.1CLZ]1^1^1^!^J$$TD$AAA8TRRRH&T&T(&T&T&TOFO0D0DFFGGG*!+- %8,%8%8999)Q!LL)O~)O)OPPP)T!LL)R.)R)RSSS %H;%H%HIII %A4%A%ABBB %LMMM$+MMOO<>> Y/// %6777 U+++%)YYu%5%5%< &z3DjQQQ(,,[999)*6FXXX P**5555M->9>9()f (;'B'B'D'D'Q ()f (;'B'B'D'D &-k<&H&H 5|55666 ;/// #5666/LV/T/T,3a393F3Fr{TZ3[3[3`0578: (=fwP^F_(`(` ({-> ? ? ? ?!LL////!LL)Z[[[%)YYu%5%5%< &z<LLL ;/// #78888$((555%j2BOTTTJJ0111J0111)*5555)*5555    % % % j*:O L L L L &'''   sresp:#""""" LLLL EMMMMM"!!!!!88888888888888 777777++++++//////@@@@@@666666666666@@@@@@777777 U   !&$    $ 8    $$$$ 5555p9999x****$     !!!!F]]]]]YYYYY     3333"6666H!! 555555p5555 ccccD;/;/;/;/~    !!!!<((((V*%%%%%PB2    ($' ' ' ' T$&                .))))6    ::::00000000D8mmmm` z *TTVV  s + 99