+
    Xi2n             	        a  0 t $ R t^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RIHt ^ RIHtHtHtHtHt ^ RIHtHtH t  ^ RI!H"t"H#t# ^ RI$H%t% ^ R	I&H't' ^ R
I(H)t)H*t* ^ RI+H,t, ^ RI-H.t. ^ RI/H0t0 ^ RI1H2t3 ^ RI4H5t6 ^ RI7H8t8 R.t9]9^ ,          t:Rt;Rt<]Pz                  P}                  RR4      t?]Pz                  P}                  RR4      t@]Pz                  P}                  RR4      P                  4       P                  4       R9   tC]Pz                  P}                  RR4      P                  4       P                  4       R9   tDRtER]FR&   ]! ]G4      P                  4       P                  tJ]JR,          tK]JR,          tL]JR ,          tM^tNR!tOR"tPR#tQ. R$NR%NR&NR'NR(NR)NR*NR+NR,NR-NR.NR/NR0NR1NR2NR3NR4NR5NR6NR7NR8NR9NR:NR;NR<NR=NR>NR?NR@NRANRBNRCNRDNRENRFNRGNRHNRINRJNRKNtR]S! ]R4      tTR!tU^-tV^tWRLtXRtYR!tZR!t[^Zt\^t]^t^]Pz                  P}                  RMR4      P                  4       ;'       g    Rt_RN RO lt`RP RQ ltaRR RS ltbRT RU ltcRV RW ltdRX RY lteRZ R[ ltfR\ R] ltg ! R^ R_4      thR` Ra ltiRb Rc ltjRd Re ltkRf Rg ltlRh Ri ltmRRjRkRlR!Rm]VRnR/Ro Rp llltnRq Rr ltoRs Rt ltp]V]X^3Ru Rv lltqRw Rx ltrRRy Rz llts ! R{ R|]4      ttR} R~ ltuR R ltvR R ltwR R ltxR tyR R ltzR R lt{R R lt|R R lt}R R lt~RR R lltR R ltR R ltRR R lltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltRR R lltR R ltR R ltR tR R ltR R ltR R ltR R ltRRRRRR/R R lltRR R lltRR#/R R lltR R ltR R ltR R ltR R lt]R8X  d
   ]! 4        R# R# )a  
RW_Site_Scraper-Orders_Page.py
==============================

Scrapes Red Wing footwear catalog:
- https://order.redwingshoes.com/footwear-rwbr

Test mode:
- Set ONLY_STYLE at the top of this file to a style number (e.g. "400")
  to scrape just that single product.

Outputs (next to this script):
- RW_Orders_Site_Scrape.md
- RW_Site_Scraper_v2_checkpoint.json   (resume state)
- RW_Site_Scraper_v2_errors.txt        (links that failed repeatedly)

Stability features:
- `safe_get()` uses short timeouts + window.stop() so Selenium doesn't hang forever
- HTTP fallback (no Selenium) for the few product pages that still time out
- Optional salvage pass at the end to retry hard failures (HTTP-first)

Cross-platform (Windows + Linux Cinnamon):
- Headless Firefox (default)
- Geckodriver resolution "like Parts_Auto" (explicit Service path; no Selenium Manager):
    1) GECKODRIVER_PATH env var (file or directory)
    2) geckodriver(.exe) on PATH
    3) auto-download geckodriver (GitHub releases) into a user cache dir
    4) (optional) if double-click/no terminal and Tk is available, prompt to pick geckodriver

Feature columns are 1/0 (not Yes/No).
Includes Brand (string) + brand family flags (Red Wing / Irish Setter / Worx).

Dependencies:
- Python 3.9+
- Firefox installed
- Selenium installed:
    Linux Mint/Ubuntu: sudo apt install -y python3-selenium
    Windows: python -m pip install selenium

Notes for Linux Mint PEP 668:
- Prefer `python3-selenium` from apt (as above).
- This script does NOT require webdriver-manager.
)annotationsN)
HTMLParser)Path)DictListOptionalSetTuple)parse_qsurljoinurlparse)Requesturlopen)BeautifulSoup)	webdriver)TimeoutExceptionWebDriverException)By)Keys)Options)Service)expected_conditions)WebDriverWaitz,https://order.redwingshoes.com/footwear-rwbrzorder.redwingshoes.comz/footwear-rwbrRW_SITE_USERNAMEzrwss614@redwingshoes.comRW_SITE_PASSWORDzWelcomeBack99!RW_HEADLESS1RW_REFRESH_LINKS str
ONLY_STYLEzRW_Orders_Site_Scrape.mdz"RW_Site_Scraper_v2_checkpoint.jsonzRW_Site_Scraper_v2_errors.txtTRED WING FOR BUSINESSFStyle #NameURLImageBrandMaleFemaleRed WingIrish SetterWorxz
Safety Toez	Steel ToezNon-Metallic ToezAluminum ToezMetatarsal GuardzSoft Toe
Waterproof
InsulationzSlip ResistantzElectrical HazardzPuncture ResistantzStatic DissipativezAnkle Protectionu   BOA® Lacing SystemDefined HeelzAll Leather Upper
ResoleablezOxford/AthleticChukkaHiker5"6"7"8"9"10"11"12"zBuilt in USAzMade in USAg      ?GECKODRIVER_PATHc                    V ^8  d   QhRRRR/# )   vboolreturnr    )formats   "wC:\Users\jasti\Documents\My_Sync\Shared\61_RW_Site\Pulled_Info\Boot_Features\RW_Scrapers\RW_Site_Scraper-Orders_Page.py__annotate__rC      s      4 C     c                    V '       d   R # R# )r   0r@   )r=   s   &rB   b01rG      s    3rD   c                    V ^8  d   QhRRRR/# r<   sr   r?   r@   )rA   s   "rB   rC   rC      s      c c rD   c                    T ;'       g    R P                  RR4      p \        P                  ! RRV 4      P                  4       p V # )r   |z\|\s+ )replaceresubstrip)rJ   s   &rB   md_escape_cellrS      s:    	
b#u%A
vsA$$&AHrD   c               $    V ^8  d   QhRRRRRR/# )r<   pathr   textr   r?   Noner@   )rA   s   "rB   rC   rC      s!      t 3 4 rD   c                    V P                  V P                  R ,           4      pVP                  VRR7       VP                  V 4       R# )z.tmputf-8encodingN)with_suffixsuffix
write_textrO   )rU   rV   tmps   && rB   atomic_writer`      s7    


4;;/
0CNN4'N*KKrD   c                   V ^8  d   QhRR/# )r<   r?   r   r@   )rA   s   "rB   rC   rC      s     <
 <
 <
rD   c                    \         P                  4       '       g4   R \        R\        R. R. R. R/ RRR/ R	. R
\        P                  ! 4       RR/#  \
        P                  ! \         P                  RR7      4      p \        V P                  R ^ 4      4      pV\        8w  d[    \         P                  \         P                  RV R24      4       R \        R\        R. R. R. R/ R
\        P                  ! 4       RR/# V P                  R \        4       V P                  R\        4       V P                  R. 4       V P                  R. 4       V P                  R. 4       V P                  R/ 4       V P                  R	. 4       V P                  R
\        P                  ! 4       4       V P                  RR4       V #   \         d     Li ; i  \         dr     \         P                  \         P                  R4      4       M  \         d     Mi ; iR \        R\        R. R. R. R/ RRR/ R	. R
\        P                  ! 4       RR/u # i ; i)versioncatalog_urlsproduct_links
done_linksrowspreferred_namesmedia_repair_doneFfail_countshard_failed_links
started_atgeckodriver_pathNrY   rZ   z.json.vz.bakz.json.corrupt)
CHECKPOINTexistsCHECKPOINT_VERSIONCATALOG_URLStimejsonloads	read_textintgetrO   r\   	Exception
setdefault)dataold_vers     rB   load_checkpointr|      s&   )LR"Br2$))+
 	
-
zz*...@Adhhy!,-(("":#9#9GG9D:Q#RS -brdiik"D	 	 		#565,b)#r*+R0diik2*D1+  ,  
	z55oFG 		 )LR"Br2$))+
 	

sb   AG	  ,F8 ,G	 9B>G	 8GG	 GG	 	I(G>=I>H	IH6IIc                    V ^8  d   QhRRRR/# )r<   rz   r   r?   rW   r@   )rA   s   "rB   rC   rC      s     I I$ I4 IrD   c           	     T    \        \        \        P                  ! V ^RR7      4       R# )r<   T)indent	sort_keysN)r`   rn   rs   dumps)rz   s   &rB   save_checkpointr      s    TZZQ$GHrD   c                    V ^8  d   QhRRRR/# )r<   liner   r?   rW   r@   )rA   s   "rB   rC   rC      s     & &C &D &rD   c                    \        \        R RR7      ;_uu_ 4       pVP                  V P                  4       R,           4       RRR4       R#   + '       g   i     R# ; i)arY   rZ   
N)open
ERRORS_TXTwriterstrip)r   fs   & rB   write_errors_liner      s9    	j#	0	0A	$% 
1	0	0	0s   'AA	c                   V ^8  d   QhRR/# r<   r?   r>   r@   )rA   s   "rB   rC   rC      s      T rD   c                 4     ^ RI p R#   \         d     R# i ; i)    NTF)tkinterrx   )r   s    rB   _can_use_tkr      s     s    c                    V ^8  d   QhRRRR/# )r<   titler   r?   Optional[str]r@   )rA   s   "rB   rC   rC     s        rD   c                   \         P                  P                  4       '       g   \        4       '       g   R #  ^ R Ip^ RIHpHp VP                  4       pVP                  4        VP                  RR4       VP                  RR4       VP                  V R7      pVP                  4        T;'       g    RP                  4       pV'       d   V# R #   \         d     R # i ; i)N)
filedialog
messageboxz-topmostTRW Site ScraperzCould not find geckodriver automatically.

Please select the geckodriver executable.
Windows: geckodriver.exe
Linux: geckodriver)r   r   )sysstdoutisattyr   r   r   r   Tkwithdraw
attributesshowinfoaskopenfilenamedestroyrR   rx   )r   tkr   r   rootrU   s   &     rB   _tk_pick_filer     s    
zz+--2uuw
D);	
 )))6

!!#t%% s$   A:C 3C C C CCc                  B    ] tR tRtRtR R ltRR R lltR R ltR	tR
# )ProgressReporteri!  zDTTY progress bar, or Tk window if launched by double-click (no TTY).c                    V ^8  d   QhRRRR/# )r<   totalrv   rl   floatr@   )rA   s   "rB   rC   ProgressReporter.__annotate__$  s     ! !c !u !rD   c                	   \        \        V4      ^4      V n        W n        \        P
                  P                  4       V n        RV n        RV n	        RV n
        RV n        V P                  '       EgN   \        4       '       Ed;    ^ RIp^ RIHp VP                  4       V n	        V P                  P!                  R4       V P                  P#                  R4       V P                  P%                  RR4       VP'                  V P                  RRR7      V n
        V P                  P)                  R	^RR
7       VP+                  V P                  V P                  RR7      V n        V P                  P)                  ^RR7       RV n        V P                  P-                  4        V P                  P/                  4        R# R# R#   \0         d    RT n         R# i ; i)   FN)ttkr   620x150zStarting...w)rV   anchorx)fillpadxpadyiD  )maximumlength)r   r   T)      )r   
   )maxrv   r   rl   r   r   r   use_ttygui_root_label_pbarr   r   r   r   r   geometry	resizableLabelpackProgressbarupdate_idletasksupdaterx   )selfr   rl   r   r   s   &&&  rB   __init__ProgressReporter.__init__$  sU   UQ'
$zz((*

+--!$'UUW


  !23

##I.

$$UE2 hhtzzchR  c A __TZZTW_X


Rg6

++-

!!#% #0&  ! !s   D7G GGc               $    V ^8  d   QhRRRRRR/# )r<   currentrv   noter   r?   rW   r@   )rA   s   "rB   rC   r   C  s!      c  d rD   c                	   \        ^ \        \        V4      V P                  4      4      pV P                  '       d   V P
                  '       d   V RV P                   RV 2P                  4       pV P                  '       d   V P                  P                  VR7       V P                  '       d   WP                  R&   V P
                  P                  4        V P
                  P                  4        R# ^ pWP                  ,          p\        \        WT,          4      4      pRV,          RWF,
          ,          ,           p\        \        P                  ! 4       V P                  ,
          R4      pW,          p	V	R	8  d   V P                  V,
          V	,          MR
p
RV RV RV P                   RV^d,          R R\        V
4       R2pV'       d   VRV 2,          p\        P                   P#                  RVR,          ,           4       \        P                   P%                  4        WP                  8X  d@   \        P                   P#                  R4       \        P                   P%                  4        R# R# )r   /z  )rV   valueN#-g-C6?g&.>        [z]  (z5.1fz%) ETA rJ   :N   Nr   )r   minrv   r   r   r   rR   r   configr   r   r   roundrr   rl   r   r   r   flush)r   r   r   msgwidthfracfilledbarelapsedrateetas   &&&        rB   r   ProgressReporter.updateC  s   aS\4::67888


IQtzzl"TF399;C{{{""",zzz&-

7#JJ'')JJ#U4<()FlSEN33diikDOO3V< /3d{tzzG#t+#b	4::,bc$ws3xjPQRRv;C

D	)*

jj JJT"JJ !rD   c                   V ^8  d   QhRR/# r<   r?   rW   r@   )rA   s   "rB   rC   r   c  s      t rD   c                	    V P                   '       d2   V P                  '       d    V P                  P                  4        R # R # R #   \         d     R # i ; iN)r   r   r   rx   )r   s   &rB   closeProgressReporter.closec  sC    888




""$ #8  s   A AA)r   r   r   r   rl   r   r   Nr   )	__name__
__module____qualname____firstlineno____doc__r   r   r   __static_attributes__r@   rD   rB   r   r   !  s    N!>@ rD   r   c                   V ^8  d   QhRR/# r<   r?   r   r@   )rA   s   "rB   rC   rC   n  s      D rD   c                    \         P                  R 8X  d   \         P                  P                  R4      ;'       gF    \         P                  P                  R4      ;'       g    \	        \
        P                  ! 4       4      p \        V 4      R,          pM^\         P                  P                  R4      ;'       g&    \	        \
        P                  ! 4       R,          4      p \        V 4      R,          pVP                  RRR7       V# )	ntLOCALAPPDATAAPPDATARW_Site_ScraperXDG_CACHE_HOMEz.cacherw_site_scraperTparentsexist_ok)osnameenvironrw   r   r   homemkdir)baseds     rB   
_cache_dirr  n  s    	ww$zz~~n-^^	1J^^cRVR[R[R]N^J**zz~~./NN3tyy{X7M3NJ**GGD4G(HrD   c                   V ^8  d   QhRR/# )r<   r?   r   r@   )rA   s   "rB   rC   rC   y  s     # #= #rD   c                    \         '       d{   \        \         4      p V P                  4       '       d   \        V 4      # V P	                  4       '       d5   R F.  pW,          pVP                  4       '       g   K#  \        V4      u # 	  R F3  p\
        V,          pVP                  4       '       g   K(  \        V4      u # 	  R F7  p\        4       V,          pVP                  4       '       g   K,  \        V4      u # 	  \        P                  R8w  ds   \        P                  ! 4       R,          \        P                  ! 4       R,          R,          3 F/  pVR,          pVP                  4       '       g   K$  \        V4      u # 	  \        P                  ! R4      ;'       g    \        P                  ! R 4      p V '       d   V # R# )geckodriver.exegeckodriverr   binz.localN)r	  r
  )r:   r   is_filer   is_dirBASE_DIRr  r   r   r  shutilwhich)pr   candr  s       rB   _resolve_from_env_or_pathr  y  s3   !"99;;q6M88:::x<<>>t9$ ; 3$<<>>t9 3 3|d"<<>>t9 3 
ww$))+%tyy{X'='EFA}$D||~~4y  G 	]#FFv||4E'FArD   c                   V ^8  d   QhRR/# )r<   r?   zTuple[str, str]r@   )rA   s   "rB   rC   rC     s     ! !_ !rD   c                    \         P                  P                  4       p \        P                  ! 4       P                  4       pV P	                  R4      '       d.   R\        P
                  ! 4       ^ ,          9   pV'       d   R	# RR3# V P	                  R4      '       d   RV9   g   RV9   d   R
# R# V R8X  d   RV9   g   RV9   d   R# R# R# )zG
Returns (asset_contains, archive_type) matching geckodriver releases.
win64win32ziplinuxaarch64arm64darwin)win64r  )zlinux-aarch64tar.gz)linux64r  )zmacos-aarch64r  )macosr  )r   platformlowermachine
startswitharchitecture)sysplatmachis_64s      rB   _platform_asset_keyr*    s     ll  "G##%D%  --/22 55gu55'""4..$$(d?i4/..""  rD   c                    V ^8  d   QhRRRR/# )r<   dest_dirr   r?   r@   )rA   s   "rB   rC   rC     s     A A4 AD ArD   c           	         \        4       w  rRp\        VRR/R7      p\        V^R7      ;_uu_ 4       p\        P                  ! VP                  4       P                  RRR7      4      pR	R	R	4       XP                  R
. 4      pR	pR	p	V FI  p
V
P                  RR4      pW9   g   K  VP                  V4      '       g   K6  V
P                  R4      pTp	 M	  V'       g   \        RV RV R24      hW	,          p\        VRR/R7      p\        V^<R7      ;_uu_ 4       p\        VR4      ;_uu_ 4       p\        P                  ! W^4       R	R	R	4       R	R	R	4       \        P                  R8X  d   RMRpW,          pVR8X  d   \        P                   ! VR4      ;_uu_ 4       pVP#                  4        FI  pVP                  V4      '       g   K  VP%                  VV R7       V V,          pVP'                  V4        M	  R	R	R	4       M\(        P                  ! VR4      ;_uu_ 4       pVP+                  4        Fu  pVP                  P                  RV,           4      '       g   VP                  V8X  g   K>  VP%                  VV R7       V VP                  ,          pVP'                  V4        M	  R	R	R	4        VP-                  RR7       \        P                  R8w  dx    \        P0                  ! V4      p\        P2                  ! VVP4                  \0        P6                  ,          \0        P8                  ,          \0        P:                  ,          4       VP=                  4       '       g   \        R4      hV#   + '       g   i     EL; i  + '       g   i     ELN; i  + '       g   i     ELZ; i  + '       g   i     EL; i  + '       g   i     EL#; i  \.         d     EL!i ; i  \.         d     Li ; i)z\
Downloads and extracts latest geckodriver into dest_dir.
Returns path to extracted driver.
z@https://api.github.com/repos/mozilla/geckodriver/releases/latest
User-AgentzRW_Site_Scraper/1.0headerstimeoutrY   rO   errorsNassetsr   r   browser_download_urlz'Could not find a geckodriver asset for r   z).wbr   r	  r
  r  r)rU   zr:gzr   T
missing_okz5Download succeeded but geckodriver was not extracted.)r*  r   r   rs   rt   readdecoderw   endswithRuntimeErrorr   r  copyfileobjr   r   zipfileZipFilenamelistextractrO   tarfile
getmembersunlinkrx   statchmodst_modeS_IXUSRS_IXGRPS_IXOTHro   )r,  	asset_keyarchive_typeapireqr8  rz   r5  dl_urldl_namer   r   archive_pathreq2r   driver_nameextracted_pathzmemberr  tsts   &                     rB   _download_latest_geckodriverr[    s   
 23I
LC
#.CD
EC	b	!	!Qzz!&&(//')/DE 
" XXh#FFGuuVR |!<!<UU12FG  DYKrR^Q__abcc%L6L2G#HID	r	"	"alD)A)AQ1  *B	" (*ww$#MK+Nu__\3//1**,??;//IIf8I4 6)AIIn- ' 0/ \\,//1,,.;;''k(9::fkk[>XIIf8I4 6;;.AIIn- ) 0t, 
ww$	(BHH^RZZ$,,%>%MPTP\P\%\]   ""RSSq 
"	!	!$ *B)A)A	"	"	" 0// 0//    		sy   6M9N!1N	N!(N51N5%A
O	4;O	8O  A6O/ 9N
	N	N!!N2	5O		O	O,+O,/O=<O=c                    V ^8  d   QhRRRR/# )r<   ckr   r?   r   r@   )rA   s   "rB   rC   rC     s     !
 !
4 !
C !
rD   c                R   \        4       pV'       d"   \        V4      P                  4       '       d   V# V P                  R 4      ;'       g    RP	                  4       pV'       d"   \        V4      P                  4       '       d   V#  \        4       p\        V4      p\        V4      V R &   \        V 4       \        V4      #   \         d\   p\        R4      pT'       d6   \        T4      P                  4       '       d   Y`R &   \        T 4       Tu Rp?# \        RT 24      hRp?ii ; i)rm   r   z$Select geckodriver / geckodriver.exeNzUnable to locate or install geckodriver.

Fix options:
  - Put geckodriver on PATH
  - OR set GECKODRIVER_PATH to the full driver path
  - OR install via package manager (Linux often: sudo apt install firefox-geckodriver)

Underlying error: )r  r   ro   rw   rR   r  r[  r   r   rx   r   r>  )r]  r  saveddestdriver_pathepickeds   &      rB   ensure_geckodriverrd    s    !#AT!W^^ VV&'--2446Ee##%%
|248!$[!1; 
EFd6l))++%+!"BM!
 "#%
 	

s%   8C   D&AD!D&D!!D&page_load_strategyeagerblock_imagespage_load_timeout
user_agentc               4    V ^8  d   QhRRRRRRRRRR	R
RRR/# )r<   r]  r   headlessr>   re  r   rg  rh  rv   ri  r   r?   zwebdriver.Firefoxr@   )rA   s   "rB   rC   rC     sN     5 555 	5
 5 5 5 5rD   c               z   \        V 4      p\        4       pV'       d   VP                  R4       \        '       d   VP                  R4       V'       d   VP	                  RV4       VP                  RR4       VP                  RR4       VP                  RR	4       VP                  R
R4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR	4       VP                  RR	4       V'       d   VP                  R^4       V'       d   VP                  RV4       \        VR7      p\        P                  ! WR7      p	V	P                  V4       V	P                  \        4       V	# )a6  Start Firefox using an explicit geckodriver path (no Selenium Manager).

Args:
    page_load_strategy: "eager" returns after DOMContentLoaded. "normal" waits for full load.
    block_images: If True, blocks images to reduce load stalls/timeouts.
    page_load_timeout: Seconds for Selenium navigation timeout.
z	-headlessz-privatepageLoadStrategyzdom.webnotifications.enabledFzmedia.volume_scalez0.0z!browser.privatebrowsing.autostartTznetwork.http.http3.enablezbrowser.cache.disk.enablezbrowser.cache.memory.enablezbrowser.cache.offline.enableznetwork.http.use-cachezplaces.history.enabledzsignon.rememberSignonsz"privacy.trackingprotection.enabledz)privacy.trackingprotection.pbmode.enabledzpermissions.default.imagezgeneral.useragent.override)executable_path)serviceoptions)rd  r   add_argumentPRIVATE_BROWSER_MODEset_capabilityset_preferenceFirefoxServicer   Firefoxset_page_load_timeoutset_script_timeoutSCRIPT_TIMEOUT)
r]  rk  re  rg  rh  ri  geckorp  ro  drivers
   &&$$$$    rB   create_driverr|    s     r"EiG[)Z(13EF 95A/7>E6>6>8%@95A3U;3U;3U;?FFM:A>;ZHU3Gw@F
  !23
n-MrD   c                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   W  s      4 rD   c                     V P                  4         V P                  R4       V P                  R4       R#   \         d     L2i ; i  \         d     R# i ; i)z;Clear cookies/storage for a clean private session baseline.about:blankz}
            try { localStorage.clear(); } catch (e) {}
            try { sessionStorage.clear(); } catch (e) {}
            N)delete_all_cookiesrx   rw   execute_script)r{  s   &rB   reset_browser_stater  W  s^    !!#	

=!	
	    s    7 "A AAAAc                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   m  s      t rD   c                     \         w  r \        P                  ! \        P                  ! W4      4       R #   \
         d    \        P                  ! T 4        R # i ; ir   )REQUEST_DELAY_RANGErr   sleeprandomuniformrx   )lohis     rB   _polite_delayr  m  s=     FB

6>>")* 

2s   *6 !AAc          
     ,    V ^8  d   QhRRRRRRRRRR	/# )
r<   urlr   r2  rv   settler   	max_triesr?   rW   r@   )rA   s   "rB   rC   rC   u  sC     C C	C C 	C
 C 
CrD   c                   Rp\        ^V^,           4       EF  p V P                  V4       \        4         \        V RR4      pVe   \	        VR4      '       d>   \	        VP
                  R4      '       d"   \        V4      ^
,           VP
                  n        \	        VR4      '       d>   \	        VP                  R4      '       d"   \        V4      ^
,           VP                  n        T P                  T4       \        P                  ! T4        T P                  \        P                  R4        R# 	  V'       d   VhR#   \         d     Lfi ; i  \         d     L/i ; i  \          d   pTp T P#                  R4       M  \         d     Mi ; i\        P                  ! T4        T P                  \        P                  R4        Rp? R#   \         d      Rp?M!i ; iRp?i\$         d   pTp Rp?MRp?ii ; i T P                  R4       M  \         d     Mi ; i\        P                  ! R	T,          4       EK0  )
zNavigate without getting stuck on pages that never fully finish loading.

- Uses page_load_timeout
- On timeout, calls window.stop() and continues if the DOM exists
- Retries a couple times with light cleanup
Ncommand_executor_client_configr2  _connbodyzwindow.stop();r  g      ?)rangerw  r  getattrhasattrr  rv   r2  r  rx   rw   rr   r  find_elementr   TAG_NAMEr   r  r   )	r{  r  r2  r  r  last_excattemptcerb  s	   &&&&&    rB   safe_getr  u  s    )-HIM*)	((1O
V%7>>r#344ARART]9^9^47L24E))1r7++)0L0L+.w<"+<( JJsO JJv##BKK8 5 +f  K       	H%%&67 JJv##BKK8  " 	H		JJ}% 		

4'>"s   E!B/D?&'E! E?E
E!EE!EE!EE!!G?,G%/F G%FG%FG%) GG"G%!G""G%%G?2G?3G::G?HH#"H#c                    V ^8  d   QhRRRR/# )r<   r  r   r?   	List[str]r@   )rA   s   "rB   rC   rC     s      s y rD   c                
   . pV 3 F  pW!9  g   K  VP                  V4       K  	  RV 9   d-   V P                  RR^4      pW19  d   VP                  V4       V# V P                  RR^4      pW19  d   VP                  V4       V# )zOReturn a small set of URL variants (www/non-www) to dodge occasional redirects.z//www.z//)appendrO   )r  outsuu2s   &   rB   _url_variantsr    s~    DU=KKN  3[[4+>KKO
 K [[x+>KKOKrD   c               $    V ^8  d   QhRRRRRR/# )r<   r  r   r2  rv   r?   r@   )rA   s   "rB   rC   rC     s!     2 2S 23 2 2rD   c           	         \        V R RRRRR/R7      p\        W!R7      ;_uu_ 4       pVP                  4       pRRR4       XP                  R	R
R7      #   + '       g   i     L#; i)r.  z_Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36Acceptz?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zAccept-Languagezen-US,en;q=0.9r/  r1  NrY   rO   r3  )r   r   r;  r<  )r  r2  rP  r8  rz   s   &&   rB   _fetch_htmlr    sc    
{W/
C 
	&	&!vvx 
' ;;wy;11 
'	&s   AA$	c                  F   a  ] tR tRtRt0 RmtV 3R ltR tR tR t	Rt
V ;t# )	_VisibleTextExtractori  z;Very small HTML->visible text extractor (no external deps).c                	2   < \         SV `  4        . V n        R # r   )superr   parts)r   	__class__s   &rB   r   _VisibleTextExtractor.__init__  s     "
rD   c                	~    VP                  4       V P                  9   d   V P                  P                  R 4       R# R# r   Nr#  _BLOCK_TAGSr  r  )r   tagattrss   &&&rB   handle_starttag%_VisibleTextExtractor.handle_starttag  -    99;$***JJd# +rD   c                	~    VP                  4       V P                  9   d   V P                  P                  R 4       R# R# r  r  )r   r  s   &&rB   handle_endtag#_VisibleTextExtractor.handle_endtag  r  rD   c                	    V'       d6   VP                  4       '       d   V P                  P                  V4       R # R # R # r   )rR   r  r  )r   rz   s   &&rB   handle_data!_VisibleTextExtractor.handle_data  s(    DJJLLJJd# !4rD   )r  >   r  brh1h2h3h4h5h6litdthtrdivfooterheaderarticlesection)r   r   r   r   r   r  r   r  r  r  r   __classcell__)r  s   @rB   r  r    s&    EK#$$$ $rD   r  c                    V ^8  d   QhRRRR/# r<   htmlr   r?   r@   )rA   s   "rB   rC   rC     s     
 
 
 
rD   c                >   \        4       p VP                  V 4       R P                  VP                  4      p\
        P                  ! V4      p\        P                  ! RRV4      p\        P                  ! RRV4      pVP                  4       #   \         d     L~i ; i)r   z\n{3,}z

z[\t\r]+rN   )
r  feedrx   joinr  html_libunescaperP   rQ   rR   )r  r  raws   &  rB   _html_to_textr    s    A	t ''!''
C


C
 C
&&FC
(C
&&S#
&C99;  s   B BBc                    V ^8  d   QhRRRR/# r  r@   )rA   s   "rB   rC   rC     s      C C rD   c                \   \         P                  ! R V \         P                  \         P                  ,          R7      pV'       g   R# VP	                  ^4      p\         P
                  ! RRV4      p\        P                  ! V4      p\         P
                  ! RRV4      P                  4       pV# )z<h1\b[^>]*>(.*?)</h1>flagsr   <[^>]+>rN   rM   )	rP   search
IGNORECASEDOTALLgrouprQ   r  r  rR   )r  minners   &  rB   _extract_first_h1r    sw    
		*D		8QRAGGAJEFF:sE*Ee$EFF63&,,.ELrD   c                    V ^8  d   QhRRRR/# r  r@   )rA   s   "rB   rC   rC     s     3# 3# 3# 3#rD   c                  aa R0o0 R	moR R lpR VV3R llp\         P                  ! RV \         P                  \         P                  ,          R7      pV'       d(   V! VP	                  ^4      4      pV! V4      '       d   V# \         P
                  ! RV \         P                  \         P                  ,          R7       F,  pV! VP	                  ^4      4      pV! V4      '       g   K*  Vu # 	  \        V 4      # )
zExtract a likely product title from HTML.

Many Red Wing product pages place the actual product name in <h3>, while <h1>
can be a site/banner header (e.g., 'RED WING FOR BUSINESS'). We therefore
try meaningful <h3> first, then fall back to <h1>.
r!   c                    V ^8  d   QhRRRR/# )r<   r  r   r?   r@   )rA   s   "rB   rC   ,_extract_first_heading.<locals>.__annotate__  s      c c rD   c                    \         P                  ! R RV 4      p \        P                  ! V 4      p \         P                  ! RRV 4      P	                  4       p V # )r  rN   rM   )rP   rQ   r  r  rR   )r  s   &rB   _clean&_extract_first_heading.<locals>._clean  sD    z3.!!%(vsE*002rD   c                    V ^8  d   QhRRRR/# r<   rJ   r   r?   r>   r@   )rA   s   "rB   rC   r  #  s      s t rD   c                   < T ;'       g    R P                  4       p V '       g   R# V P                  4       pVS9   g   VS9   d   R# \        V4      ^8:  d   R# \        P                  ! RV4      '       g   R# R# )r   Fz[A-Z]T)rR   upperlenrP   r  )rJ   upBADSTOPs   & rB   _ok#_extract_first_heading.<locals>._ok#  s[    WW"OOWWY9d
r7a<yy2&&rD   z<td[^>]*class=['\"][^'\"]*prTitle[^'\"]*['\"][^>]*>\s*Name\s*</td>\s*<td[^>]*class=['\"][^'\"]*prValue[^'\"]*['\"][^>]*>(.*?)</td>r  z<h3\b[^>]*>(.*?)</h3>>	   CARESIZINGDETAILSREVIEWSFEATURES
TECHNOLOGYSPECIFICATIONS
SIZE & FITRELATED PRODUCTS)rP   r  r  r  r  finditerr  )r  r  r  mnamerJ   mh3r  r  s   &     @@rB   _extract_first_headingr    s     #
#CD
  II	Immbii'	E 5;;q>"q66H {{3TQSQZQZAZ[399Q< q66H \
 T""rD   c                    V ^8  d   QhRRRR/# )r<   hrefr   r?   r>   r@   )rA   s   "rB   rC   rC   I  s     " "3 "4 "rD   c                ^   T ;'       g    R P                  4       p V '       g   R# V P                  4       pVP                  R4      '       g/   VP                  R4      '       g   VP                  R4      '       d   R# RV9   d   R#  \        V 4      pVP                  ;'       g    R P                  4       P                  R4      p\        VP                  ;'       g    R 4      pRV9   dj   \        P                  ! R	V4      '       dK   R FB  pVP                  V4      '       g(   VP                  VP                  4       4      '       g   KA   R# 	  R# R# R
V9   d    \        P                  ! RV4      '       d   R# \        P                  ! RV4      '       d   R# R#   \         d    Tp/ p Li ; i)r   Fjavascript:zmailto:ztel:z/safety-boot/Tr   
/footwear-z.*/footwear-[^/]+z	/product/z\d{3,6}z/\d{3,6}\.html$stylestyleNumberskuitempid	productId)rR   r#  r%  r   rU   r   r
   queryrx   rP   	fullmatchrw   r  )r  lowparsedrU   r  keys   &     rB   _is_product_linkr  I  sO   JJBD
**,C
~~m$$y(A(AS^^TZE[E[#$!!r((*11#6++,
 t<<,d33R99S>>UYYsyy{%;%; S  dryyT:: 
yy#T**+  s   F $7F F F,+F,c                   . p\        4       p V P                  \        P                  R4      pV Fr  p VP	                  R4      ;'       g    RP                  4       pV'       d   WR9   d   K=  \        V4      '       g   KP  VP                  V4       VP                  V4       Kt  	   V P                  \        P                  R4      pV Fr  p VP	                  R4      ;'       g    RP                  4       pV'       d   WR9   d   K=  \        V4      '       g   KP  VP                  V4       VP                  V4       Kt  	  V'       g    V P                  \        P                  R4       Fr  p VP	                  R4      ;'       g    RP                  4       pV'       d   WR9   d   K=  \        V4      '       g   KP  VP                  V4       VP                  V4       Kt  	  V# V#   \         d    Rp ELi ; i  \         d     EL^i ; i  \         d    Rp EL"i ; i  \         d     Li ; i  \         d    Rp Li ; i  \         d     T# i ; i)u  Locate product anchors on the catalog page.

The Red Wing wholesale catalog renders product tiles with specific CSS
classes.  Each tile (<li>) contains two links pointing at the same
product detail page: one wrapping the image and one wrapping the product
name.  Both have an href that ends in ".html" with the numeric style
number.  On the current site the anchor for the name has the classes
``c-product-tile__pdp-link js-product-name``, while the image link has
no special class but is still within the tile.  This helper first
attempts to extract anchors using those specific selectors and falls
back to scanning all anchors when necessary.

Returns a list of WebElement objects (anchors) with unique hrefs.
Hli.js-product-grid-item a.c-product-tile__pdp-link.js-product-name[href]r  r   (li.js-product-grid-item a[href$='.html']z
//a[@href])setfind_elementsr   CSS_SELECTORget_attributerR   rx   r  r  addXPATH)r{  anchorsseenelementsr   r  	elements2s   &      rB   _find_product_anchorsr$  n  s    GUD''OOV
 A/552<<> 4<%%q! ((OOG
	 A/552<<> 4<%%q!  	))"((LAOOF399r@@BD t|#D))NN1%HHTN B N7NS         ! D  	N	s   %G6 G"G"G6 %G6  &G6 '%H H%H6H >H &H #I  ,H-H-I  I  8&I  "G3/G6 2G33G6 6HHHH HH H*)H*-H=:I  <H==I   IIc                   V ^8  d   QhRR/# r<   r?   r  r@   )rA   s   "rB   rC   rC     s     = =y =rD   c                6    V P                   ;'       g    \        p V P                  ;'       g    RpV'       g   . # \	        4       p \        VR4      pVP                  R4       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  VP                  R4       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  VP                  RRR7       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  \        P                  ! V4      p. R
OpV Ft  p	\        P                  ! W\        P                  R	7       FG  p
V
P!                  ^ 4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  Kv  	  . p\	        4       pV FK  p\#        W4      p\%        V4      '       g   K!  W9   d   K)  VP                  V4       VP'                  V4       KM  	  V#   \         d    \        p ELUi ; i  \         d    Rp ELRi ; i  \         d     EL-i ; i)zIFallback link discovery when clickable anchors are sparse or JS-rendered.r   html.parserr  r  r  r   T)r  r  )zhttps?://[^\s\"'<>]+z2/(?:footwear-rwbr|safety-boot|product)/[^\s\"'<>]+z"/\d{3,6}\.html(?:[?#][^\s\"'<>]*)?)current_urlCATALOG_URL_PRIMARYrx   page_sourcer  r   selectrw   rR   r  find_allr  r  rP   r  r  r  r   r  r  )r{  base_urlr  
candidatessoupr   r  blobpatternspatr  r  outr!  r  abs_urls   &               rB   '_extract_product_links_from_page_sourcer6    s*   '%%<<)<!!''R 	5JT=1ghAEE&M''R..0Dtt$ i
 GHAEE&M''R..0Dtt$ I
 s.AEE&M''R..0Dtt$ / T"DH
 Sbmm<A!!r((*Aqq! =  CUD((((?

7  Js  '&'
  4  sd   I I I5 I5 7J	  J	 A J	 J	 7AJ	 :J	 J	 I21I25JJ	JJc                    V ^8  d   QhRRRR/# )r<   
target_urlr   r?   rW   r@   )rA   s   "rB   rC   rC     s     a a a arD   c                T   \        4        \        W4       \        V 4       \        P                  R3\        P                  R3\        P                  R3\        P                  R3\        P                  R3\        P                  R3.p\        P                  R3\        P                  R3\        P                  R	3\        P                  R
3.p\        P                  R3\        P
                  R3\        P
                  R3.p \        V 4      '       d   R#  RpRpV F1  w  rx V P                  Wx4      p	V	P                  4       '       d   T	p MK3  	  V F1  w  rx V P                  Wx4      p	V	P                  4       '       d   T	p MK3  	  V'       d	   V'       g   R#  VP                  4        VP                  \        4        VP                  4        VP                  \        4       Rp
V FY  w  rx V P                  Wx4      pVP                  4       '       d-   VP                  4       '       d   VP                  4        Rp
 MKY  K[  	  V
'       g    VP                  \         P"                  4       \$        P$                  ! 4       ^,           p\$        P$                  ! 4       V8  dp   \        V 4        \        V 4      '       d   R#   V F.  w  rxV P                  Wx4      pVP                  4       '       g   K.   M	  R# \$        P&                  ! R4       K  R#   \         d     ELi ; i  \         d     EK  i ; i  \         d     EK  i ; i  \         d     ELi ; i  \         d     ELi ; i  \         d     EK  i ; i  \         d     Li ; i  \         d     R# i ; i)zCAttempt a best-effort login for order.redwingshoes.com if prompted.zinput[type='email']zinput[name='email']zinput[name='username']zinput[id*='email' i]zinput[id*='user' i]zinput[autocomplete='username']zinput[type='password']zinput[name='password']zinput[id*='password' i]z&input[autocomplete='current-password']zbutton[type='submit']z//button[contains(translate(.,'LOGINSGNIN','loginsgnin'),'login') or contains(translate(.,'LOGINSGNIN','loginsgnin'),'sign in')]z//input[@type='submit']NFTg?)r  r  dismiss_popupsr   r  r  r$  rx   r  is_displayedclear	send_keysr   r   
is_enabledclickr   ENTERrr   r  )r{  r8  login_user_selectorslogin_pass_selectorssubmit_selectorsuser_elpass_elbyselr  	submittedbtnendpes   &&            rB   ensure_logged_inrL    s_   OV 6 
/0	/0	23	01	/0	:; 
23	23	34	BC	 
12	  V  	W	,- (( )
 GG'	&&r/D  "" # ( (	&&r/D  "" # ( ' &' &'I#	%%b.C!!cnn&6&6		 	 '7! $ $**% ))+
C
))+
v	$V,, -	/((1??$$ 0
  	

3! g    		  		  
    		  		  		s   L %)L*)L=M M" 3<M40M43N +N 8N :N L'&L'*L:9L:=MMMM"M10M14NNNNN'&N'c                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   X  s      T rD   c                6   \         P                  R3\         P                  R3\         P                  R3\         P                  R3\         P                  R3.pV F  w  r# \	        V ^
4      P                  \        P                  ! W#34      4      pV P                  RV4       \        P                  ! R4        VP                  4        \        P                  ! R	4        R
# 	  R
#   \         d    T P                  RT4        L<i ; i  \         d     K  i ; i)zCOpen the Footwear section after login when required by the site UI.zfootwear-labelz#footwear-labelz;//button[@id='footwear-label' or @aria-controls='footwear']z//a[@id='footwear-label']zU//*[self::button or self::a or @role='tab'][contains(normalize-space(.), 'Footwear')]?arguments[0].scrollIntoView({block:'center', inline:'center'});333333?arguments[0].click();g?N)r   IDr  r  r   untilECelement_to_be_clickabler  rr   r  r?  rx   )r{  	selectorsrF  rG  els   &    rB   open_footwear_sectionrX  X  s     
 !	+,	PQ	./	jkI 	vr*001K1KRI1VWB!!"ceghJJtC
 JJsO   C%%&=rBC  		s7    AD	9C&	D	&DD	DD		DDc                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   p  s     & &4 &rD   c                   \         P                  R3\         P                  R3\         P                  R3.pRpV Fq  w  r4 \        V ^4      P                  \        P
                  ! W434      4      pV P                  RV4       \        P                  ! R4        VP                  4        Rp M	   \        V \        4        \        V ^4      P                  R	 4       R
#   \         d    T P                  RT4        LUi ; i  \         d     K  i ; i  \         d     Lai ; i  \         d)     \        T \        4        R
#   \         d      R
# i ; ii ; i)zQUse Footwear > Red Wing navigation described in docs, then ensure /footwear-rwbr.z%//a[contains(@href,'/footwear-rwbr')]z0//button[contains(@data-target,'footwear-rwbr')]zx//*[self::a or self::button][contains(normalize-space(.), 'Red Wing') and not(contains(normalize-space(.), 'Heritage'))]FrO  rP  rQ  Tc                ~    \         \        V P                  4      P                  ;'       g    R P	                  4       9   # r   )FOOTWEAR_RWB_SLUGr   r)  rU   r#  r  s   &rB   <lambda>-open_red_wing_footwear_page.<locals>.<lambda>  s*    'Xamm-D-I-I-O-OR,V,V,XYrD   N)r   r  r   rS  rT  rU  r  rr   r  r?  rx   r  r*  )r{  rV  clickedrF  rG  rW  s   &     rB   open_red_wing_footwear_pagera  p  sT    
:;	EF	  N  	OI
 G	vq)//0J0JB90UVB!!"ceghJJtC
 G ",-fb!''Y	
  C%%&=rBC  		    	V01 		ss    ADC)D0D D% C?<D>C??DDDD"!D"%E1EEEEEc                   V ^8  d   QhRR/# r&  r@   )rA   s   "rB   rC   rC     s     E Ei ErD   c                    V P                   ;'       g    \        p. p. R
OpV Fo  p V P                  \        P
                  V4       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  Kq  	   V P                  \        P
                  R4       FO  pR FF  pVP                  V4      ;'       g    RP                  4       pRV9   g   K5  VP                  V4       KH  	  KQ  	   V P                  ;'       g    Rp	V	'       d   \        P                  ! V	4      p
\        P                  ! RV
\        P                  R7       F#  pVP                  VP                  ^ 4      4       K%  	  \        P                  ! RV
\        P                  R7       F#  pVP                  VP                  ^ 4      4       K%  	  . p\!        4       pV F  p\#        W4      p \%        V4      pVP&                  ;'       g    RP)                  4       pVP*                  ;'       g    RP)                  4       p\,        T9  d   Ko  \        P.                  ! RT4      '       g   K  TP0                   R	TP&                   T 2pTT9  g   K  TP3                  T4       TP                  T4       K  	  \        V9  d   VP5                  ^ \        4       V#   \         d    \        p ELi ; i  \         d     EK  i ; i  \         d     ELi ; i  \         d    Rp	 ELi ; i  \         d     EK]  i ; i)zGDiscover footwear brand catalog paths from RW site navigation/elements.r  r   z&[data-target], [data-url], [data-href]r	  z/footwear-[a-z0-9\-]+r  z)https?://[^\s\"'<>]*/footwear-[a-z0-9\-]+z/footwear-[a-z0-9\-]+$z://)za[href*='/footwear-']z[id*='footwear' i] a[href]znav a[href*='footwear'])zdata-targetzdata-urlz	data-href)r)  r*  rx   r  r   r  r  rR   r  r+  r  r  rP   r  r  r  r  r   r   netlocr#  rU   	SITE_HOSTr  schemer  insert)r{  r.  r/  dom_selectorsrG  rW  r  attrr=   r  r1  r  urlsr!  r  r  r  hostrU   norms   &                   rB   discover_footwear_catalog_urlsrm    s   '%%<<)< JM
 	**2??C@((066B==?4%%d+ A &&r8`aB@%%d+11r88:1$%%a( A b!!''R   &5t2==QAaggaj) RI4WYWdWdeAaggaj) f DUDH"	AHHNN))+DFFLLb'')D D yy2D99((3qxxj/tHHTNKK $ $&A*+KC  '&'"  		    "  		s   K' K' :K?#K?>K??L L 2L L$ L$ 9AL8;L8'K<;K<?LLL! L!$L54L58MMc                    V ^8  d   QhRRRR/# )r<   r2  rv   r?   rW   r@   )rA   s   "rB   rC   rC     s     1 1S 1$ 1rD   c                F    R R lp\        W4      P                  V4       R# )zVWait for shared footwear-page elements instead of requiring product links immediately.c                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   5wait_for_footwear_catalog_ready.<locals>.__annotate__  s     g gT grD   c                (    V P                  \        P                  R4      pV'       d   R#   V P	                  \        P
                  R4      P                  ;'       g    RP                  4       pRT9   pRT9   pRT9   pR	T9   pR
T9   pRT9   ;'       g(    RT P                  ;'       g    RP                  4       9   pT;'       d+    T;'       g!    T;'       g    T;'       g    T;'       g    T#   \         d     Li ; i  \         d     R# i ; i)u  Check that the catalog UI has loaded.

On the current Red Wing site, the grid of products is present in
``li.js-product-grid-item`` elements.  Earlier heuristics looked for
text such as "Sort By", "Results" or "Filters"; those still apply
but may not appear until the user scrolls.  We consider the catalog
ready when either at least one product tile exists or legacy text
heuristics are satisfied.
zli.js-product-grid-itemTr  r   Fresultzsort byfilterszmore resultsr  footwearr	  )	r  r   r  rx   r  r  rV   r#  r)  )	r  tilestxthas_resultshas_sorthas_filtershas_more
has_searchhas_footwear_ctxs	   &        rB   _ready/wait_for_footwear_catalog_ready.<locals>._ready  s   	OOBOO5NOE 	>>"++v6;;AArHHJC #o#3&!S(_
%,]]ATATRT@[@[@]0]ffX%e%e%e%e%e%ex%e%e[ef  		  		s(   'C1 0D D 1C?>C?DDN)r   rS  )r{  r2  r~  s   && rB   wait_for_footwear_catalog_readyr    s    g: &"((0rD   c               $    V ^8  d   QhRRRRRR/# )r<   r  r   r  r?   zTuple[str, str, str]r@   )rA   s   "rB   rC   rC     s'     C* C*3 C*S C*=Q C*rD   c                   \        V R4      p\        V4      pRpRpVP                  RRR7      ;'       g    VP                  RR7      pV'       d   VP                  R4      pVP                  R	4      pV'       dD   VP                  R
RR7      p\        P
                  ! RV4      p	V	'       d   V	P                  ^4      pV'       d   VP                  R
RR7      pV'       EgW   VP                  R4       EF@  p
 \        P                  ! V
P                  RR7      4      p\        T\        4      '       d   TMT.pT F  p\        T\        4      '       g   K  T'       g4   TP                  R4      ;'       g    RP                  4       pT'       d   TpT'       gn   R Fg  p\!        TP                  T4      ;'       g    R4      P                  4       p\        P
                  ! RT4      pT'       g   KV  TP                  ^4      p M	  T'       g   K  T'       g   K   M	  T'       g   EK6  T'       g   EKA   M	  V'       g   R F  p VP#                  V4      pV'       g   K  VP%                  R4      '       d+   VP                  R4      ;'       g    RP                  4       pMVP                  R
RR7      pV'       g   K}  RVP'                  4       9  g   K  Tp M	  V'       g   V'       d   RV 2MT;'       g    RpV'       g   VP                  4       pV'       g   TpW4V3#   \         d     EK?  i ; i  \         d    Rp Li ; i)zMExtract (style_number, name, style_text) from legacy and RW order-site pages.r(  r   r  	shoeguide)class_printSpacing)idr  strongrN   T)rR   z	#\s*(\d+)z"script[type='application/ld+json']r   \b(\d{3,6})\bNmetacontentzred wing for businessr"   )r  mpn	productID)r  r  r   zmeta[property='og:title'])r   extract_style_from_urlfindget_textrP   r  r  r,  rs   rt   rx   
isinstancelistdictrw   rR   r   
select_oner%  r#  )r  r  r0  style_numberr   
style_text	guide_divh3_tag
strong_tagmatchscriptpayloadobjsobjnmr  r  r  rG  rW  rw  s   &&                   rB    extract_style_and_name_from_htmlr    s   }-D)#.LDJ		%	4TT		^	8TI%^^H-
D9JIIlJ7E${{1~&&s$&7D4kk"FGF**V__4_%@A )$777gYD!#t,,''&///R668B!#:!#''#,"4"4"5;;=II&6<1+,771:L!  ; 4LL   t/ H2 EC__S) ~~f%%vvi(..B557kk#Tk2s.ciikA F 1=w|n-DJJB
!'')z))Y  4  s$   8&L#L6#L32L36MMc                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   G  s      d rD   c                   . ROp\         P                   ! 4       R,           p\         P                   ! 4       V8  d   RpV Ff  p V P                  \        P                  V4      pVP	                  4       '       d-   VP                  4       '       d   VP                  4        Rp MKf  Kh  	  V'       g   R# \         P                  ! R4       K  R#   \         d     K  i ; i)N//button[contains(.,'Agree') or contains(.,'Accept') or contains(.,'Proceed')]g       @FT皙?N)r  zI//a[contains(.,'Agree') or contains(.,'Accept') or contains(.,'Proceed')]uR   //button[contains(.,'Close') or contains(.,'×') or contains(@aria-label,'Close')])	rr   r  r   r  r;  r>  r?  rx   r  )r{  xpathsrJ  r`  xprW  s   &     rB   r:  r:  G  s    F
 ))+
C
))+
B((26??$$HHJ"G *9$  

3   s   ACCCCc                    V ^8  d   QhRRRR/# )r<   
max_roundsrv   r?   rW   r@   )rA   s   "rB   rC   rC   ^  s     B B3 B BrD   c           	        ^ pRp\        V4       EF  p\        V 4       RpR F  p V P                  \        P                  V4      pVP                  4       '       g   K<  VP                  R4      p\        VR4      '       d   VP                  4       '       g   Kw  VR9  d   K   V P                  RV4       \        P                  ! R4        VP                  4        Rp\        P                  ! R	4        M	  \        V 4      p	\        V	 U
u0 uF,  qP                  R
4      '       g   K  V
P                  R
4      kK.  	  up
4      pW8X  d   V^,          pM^ pTpV'       g   V^8  d    R#  V P                  R4       \        P                  ! R4       EK  	  R#   \         d     Li ; i  \         d+     T P                  RT4        L  \         d      EK  i ; ii ; i  \         d     EK  i ; iu up
i   \         d     Li ; i)r   Fdisabledr>  NrO  rP  rQ  T333333?r  z/window.scrollTo(0, document.body.scrollHeight);g      ?)zz//button[contains(translate(normalize-space(.),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'more results')]zu//a[contains(translate(normalize-space(.),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'more results')])Nfalser   )r  r:  r  r   r  r;  r  r  r>  r  rx   rr   r  r?  r$  r  )r{  r  stable_rounds
last_count_clicked_morer  rI  disabled_attrlinksr   counts   &&          rB   scroll_to_load_allr  ^  s   MJ:v
B!))"((B7'')) # 1 1* =3--cnn6F6F (;;))Y 

4 !IIK  $

3G
N &f-eWev?V,Q__V,eWXQMMJ  2	!!"ST 	

3 8 ! 
 ! !!--.EsK$ ! !	!   X  		s   6G$8G$G$$F6G$F,G$G7
*G7
-G<F)&G$(F))G$,G!8G
G$G	G!G$G	G!!G$$G43G4<H
	H
c                    V ^8  d   QhRRRR/# rI   r@   )rA   s   "rB   rC   rC     s      3 3 rD   c                $   T ;'       g    R P                  4       p \        P                  ! RRV 4      p V '       g   R # V P                  4       pVR9   d   R # \        P                  ! RV 4      '       d   R # RV 9   d   R # \        V 4      ^8  d   R # V # )r   rM   rN   z\d{2,}$>   r  r  r  r  
QUICK VIEWADD TO CARTVIEW DETAILSr!   )rR   rP   rQ   r  r  r  )rJ   r  s   & rB   _clean_listing_namer    sz    	
bA
vsAA		A 	 	 	||Iq!!
ax
1vzHrD   c                   V ^8  d   QhRR/# r<   r?   r   r@   )rA   s   "rB   rC   rC     s     1 1C 1rD   c                   . p TP                  V P                  R4      ;'       g    R4        TP                  V P                  R4      ;'       g    R4        TP                  V P                  ;'       g    R4       . pV FS  pV'       g   K  \	        V4      P                  4        F)  p\        V4      pV'       g   K  VP                  V4       K+  	  KU  	  \        4       p. pV F-  pWE9  g   K  VP                  V4       VP                  V4       K/  	  V'       g   R# R R lpVP                  VRR7       V^ ,          #   \         d     EL%i ; i  \         d     ELi ; i  \         d     Li ; i)zJBest-effort: read the product name as displayed on a catalog listing tile.
aria-labelr   r   c                    V ^8  d   QhRRRR/# )r<   rJ   r   r?   rv   r@   )rA   s   "rB   rC   /_listing_name_from_anchor.<locals>.__annotate__  s     
 
 
 
rD   c                  a  \        S 4      p\        P                  ! R S 4      '       d
   V^,          p\        P                  ! RS P                  4       4      '       d
   V^
,          p\        P                  ! RS P                  4       4      '       d
   V^,          p\        ;QJ d    V 3R lR 4       F  '       g   K   RM	  RM! V 3R lR 4       4      '       d
   V^,          pV# )z[A-Za-z]\bWOMEN'?S\b
\bMEN'?S\bc              3  H   <"   T F  qSP                  4       9   x  K  	  R # 5ir   )r#  .0rY  rJ   s   & rB   	<genexpr>;_listing_name_from_anchor.<locals>.score.<locals>.<genexpr>  s     X'W!AGGI~'Ws   "TF)viewquickcartcomparewishlist)r  rP   r  r  any)rJ   scs   f rB   score(_listing_name_from_anchor.<locals>.score  s    V99[!$$"HB99_aggi00"HB99]AGGI..!GB3X'WX333X'WXXX"HB	rD   T)r  reverse)
r  r  rx   rV   r   
splitlinesr  r  r  sort)r   r/  linesclnr!  uniqr  s   &       rB   _listing_name_from_anchorr    sK   J!//,7==2>!//'288b9!&&,,B' Ea&##%B$R(BrR  &  5DD>HHRLKKO 
 
 	II%I&7NY      sE   "E E "E E E) 8E) EEE&%E&)E76E7c               $    V ^8  d   QhRRRRRR/# )r<   rg   List[List[str]]	preferredzDict[str, str]r?   rv   r@   )rA   s   "rB   rC   rC     s!       N s rD   c                j   V '       d	   V'       g   ^ # ^ pV  F  p\        V\        4      '       d   \        V4      ^8  d   K+  V^ ,          ;'       g    RP                  4       pV'       g   KV  \	        VP                  VR4      4      pV'       g   K{  V^,          V8w  g   K  WS^&   V^,          pK  	  V# )zQOverwrite Name column using preferred mapping (by style). Returns number updated.r   )r  r  r  rR   r  rw   )rg   r  updatedr8  r  prefs   &&    rB   apply_preferred_namesr    s    yG!T""c!fqj1""$"9==#;<4AaDDLaDqLG  NrD   c                   V ^8  d   QhRR/# )r<   r?   z Tuple[List[str], Dict[str, str]]r@   )rA   s   "rB   rC   rC     s     =& =&%E =&rD   c           	     p   . p\        4       p\        4       p/ p\        V \        4       \        V 4       \	        V 4      ;'       g    \        \        4      p\        P                  P                  R\        V4       R24       V F&  p\        P                  P                  RV R24       K(  	  V EF  p\        4        \        W4       \        W4       \        V ^#R7       \        V 4       \        V 4       . p\!        V 4       FB  p	V	P#                  R4      p
V
'       g   K  \%        V
4      '       g   K1  VP'                  V
4       KD  	  \)        V 4       F  p
W9  g   K  VP'                  V
4       K  	  \        P                  P                  RV R\        V4       R24       V F  p
W9   d   K  \+        V
4      p\,        '       d   V'       d	   R	V9   d    V'       d+   W9   d   VP/                  V
4       KS  VP/                  V4       VP/                  V
4       VP'                  V
4       K  	  EK  	  W3# )
zCollect unique product links from all configured catalog sections.

Dedupes by style number when possible (preferred), otherwise by URL.
z([collect] footwear catalogs discovered: r   z[collect] catalog: r1  r  z
[collect] z -> discovered z candidate product links
zcatalog=international)r  rL  r*  rX  rm  r  rq   r   stderrr   r  r  r  r  r:  r  r$  r  r  r  r6  r  "PREFER_INTERNATIONAL_LISTING_NAMESr  )r{  	all_links	seen_href
seen_stylerh   rd   r  r  
page_linksr   r  r  s   &           rB   collect_product_linksr    s   
 I%I5J&(OV01&!1&9OOT,=OLJJ?L@Q?RRTUV

.qc45  %';v6" !#
&v.A??6*D#D))d# / <FCD%!!$' D 	

:cU/#j/9JJdefD *40E11e@W[^@^&MM$'u%MM$T"! 3 V %%rD   c                    V ^8  d   QhRRRR/# )r<   linkr   r?   r@   )rA   s   "rB   rC   rC   C  s            rD   c                   V '       g   R # \         P                  ! RV 4      pV'       d   VP                  ^4      # \         P                  ! RV 4      pV'       d   VP                  ^4      # \         P                  ! RV \         P                  R7      pV'       d   VP                  ^4      # \         P                  ! RV \         P                  R7      pV'       d   VP                  ^4      # \         P                  ! RV 4      pV'       d'   RV P	                  4       9   d   VP                  ^4      #  \        \        V 4      P                  4      pR	 F  pVP                  V4      ;'       g*    VP                  VP	                  4       4      ;'       g    . pV F@  p\         P                  ! R\        V4      4      pV'       g   K-  VP                  ^4      u u # 	  K  	  R #   \         d     R # i ; i)
r   z/safety-boot/(\d+)[-/]z/safety-boot/(\d+)z2/footwear-[^/]+/(?:[^/?#]*/)?(\d{3,6})(?:[-/?#]|$)r  z/(\d{3,6})\.html(?:[?#]|$)z/(\d{3,6})(?:[-/?#]|$)zfootwear-rwbrr  r
  )rP   r  r  r  r#  r
   r   r  rw   r   rx   )r  r  qr  valsr=   s   &     rB   r  r  C  sn   
		+T2Awwqz
		'.Awwqz 			GUWUbUbcAwwqz
		/R]]KAwwqz
		+T2A_

,wwqz	Xd^))*NC55:99syy{!399rDII.A71771:%  O   s*   1:G" ,&G" -G" G" G" "G10G1c                    V ^8  d   QhRRRR/# r  r@   )rA   s   "rB   rC   rC   h  s        rD   c                  a  S ;'       g    R P                  4       P                  4       o S '       g   R# Rp\        ;QJ d    V 3R lV 4       F  '       g   K   RM	  RM! V 3R lV 4       4      '       d   R# \        S 4      ^<8  d   R# R# )r   Tc              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r@   r  s   & rB   r  #_looks_like_junk.<locals>.<genexpr>n  s     
';a6;   F)zwindow.openr  zfacebook.comzhttp://zhttps://)rR   r#  r  r  )rJ   junk_tokenss   f rB   _looks_like_junkr  h  s^    	
b!AWK
s
';
'sss
';
'''
1v{rD   c                    V ^8  d   QhRRRR/# )r<   rowr  r?   r>   r@   )rA   s   "rB   rC   rC   v  s      Y 4 rD   c                     V ^ ,          ;'       g    RP                  4       pV ^,          ;'       g    RP                  4       pT'       g   R# TP                  4       \        8X  d   R# R#   \         d     R# i ; i)r   r   TF)rR   rx   r  BAD_NAME_SENTINEL)r  r  r   s   &  rB   _is_bad_rowr  v  sh    Q2$$&A"##% zz|((  s   A+  A+ A+ +A:9A:c                    V ^8  d   QhRRRR/# r<   r]  r   r?   rv   r@   )rA   s   "rB   rC   rC     s     # #T #c #rD   c                   V P                  R4      '       d   ^ # V P                  R. 4      ;'       g    . p\        V P                  R. 4      ;'       g    . 4      p^ p\        4       pV F  p\        V\        4      '       d	   V'       g   K#  V^ ,          ;'       g    RP	                  4       pV'       g   KN  \        V4      \        8  d   VP                  V4       Ku  \        V4      ^8  d"   V^,          ;'       g    RP	                  4       MRp\        V4      ^8  d"   V^,          ;'       g    RP	                  4       MRpV'       d   V'       d   K  VP                  V4       EK  	  V'       g   ^ # \        V4       F:  p	\        V	4      p
V
'       g   K  W9   g   K   VP                  V	4       V^,          pK<  	  \        V4      V R&   V# )zIf checkpoint rows are missing URL/Image columns or have empty URL/Image, requeue those links.
Runs at most once per checkpoint unless you delete/clear ck['media_repair_done'].
ri   rg   rf   r   )rw   r  r  r  rR   r  EXPECTED_COLSr  r  discardsorted)r]  rg   rf   removedstyles_neededr8  r  url_cellimg_cellr  rZ  s   &          rB   repair_missing_mediar    sd    
vv!""66&"##DRVVL"-334JGEM!T""!1""$q6M!e$+.q6A:AaDJJB%%'2+.q6A:AaDJJB%%'2xxe$  Z #D)2"%t$qLG	 ! j)B|NrD   c                    V ^8  d   QhRRRR/# r  r@   )rA   s   "rB   rC   rC     s     - -4 -C -rD   c                    \        V P                  R. 4      ;'       g    . 4      p\        V P                  R. 4      ;'       g    . 4      pV'       d	   V'       g   ^ # V Uu0 uF(  q3'       g   K  \        V4      '       g   K  V^ ,          kK*  	  ppV'       g   ^ # ^ p\        V4       F:  p\	        V4      pV'       g   K  Wt9   g   K   VP                  V4       V^,          pK<  	  V'       d   \        V4      V R&   \        V P                  R/ 4      ;'       g    / 4      p\        VP                  4       4       F2  p	\	        V	4      pV'       g   K  Wt9   g   K   VP                  V	R4       K4  	  WR&   V P                  R. 4      ;'       g    .  U
u. uF  p
\	        V
4      V9  g   K  V
NK  	  up
V R&   \        V 4       V# u upi u up
i   \         d     ^ # i ; i)aM  If checkpoint contains obviously bad rows, un-mark those links as 'done'.

This fixes the situation where a previous run captured the site header/ads into the Name/Brand
columns and those rows are now 'stuck' because resume logic skips already-done links.

Returns:
    Number of links that were re-queued (removed from done_links).
rg   rf   rj   Nrk   )r  rw   r  r  r  remover  r  keyspopr   rx   )r]  rg   rf   r8  
bad_stylesr  r  rZ  fckr  s   &          rB   repair_bad_checkpoint_rowsr    s   $BFF62&,,"-b177R8
:$(ADqAd+a.daddD
A$D'-Brb&!!$'1	 % %j1B| bff]B/5526B"'')_+A.2"*FF1dO % !#} 4666:Mr3R3X3XVX3X 'V3XQ)?)B*)T ()q3X 'VB"# B9 B.'V  s   G $G G G G "	G0GG
G  G G 	%G /+G 3G G 2G G G*G0G 
G GGc               $    V ^8  d   QhRRRRRR/# )r<   
text_blockr   
field_namer?   r@   )rA   s   "rB   rC   rC     s!       # # rD   c           
        T ;'       g    RP                  4        Uu. uF  q"P                  4       NK  	  pp\        V4       F  w  rEV'       g   K  VP                  V4      '       g   K(  V\	        V4      R P                  R4      pV'       d   Vu # \        V^,           \        V^,           \	        V4      4      4       F1  pW7,          ;'       g    RP                  4       pV'       g   K-  Vu u # 	   R# 	  R# u upi )u   Parse a simple field/value from extracted page text.

Red Wing pages often render fields in tables so the extracted text looks like:

    Name
    DynaForce®

We support both 'Name: DynaForce' and 'Name' on one line with the value on the next.
r   Nz :	)r  rR   	enumerater%  r  r  r   )	r  r  r  r  ir   tailjnxts	   &&       rB   parse_field_liner    s     $.#3#3"?"?"AB"ABXXZ"AEBU#??:&&J()//7D1q5#a!eSZ"89x~~2,,.3J :  $  Cs   C<c               $    V ^8  d   QhRRRRRR/# )r<   header_textr   	body_textr?   r@   )rA   s   "rB   rC   rC     s!      s s s rD   c                   T ;'       g    R P                  4       p\        P                  ! RV\        P                  R7      pV'       d   VP	                  ^4      P                  4       pV'       db   . pVP                  4        F:  pTP                  VP                  4       '       d   TMVP                  4       4       K<  	  RP                  V4      # VP                  4        F  p\        P                  ! RV\        P                  R7      '       g   K2  \        P                  ! RVP                  4       \        P                  R7      pV'       d   VP	                  ^4      P                  4       pV'       dd   . pVP                  4        F:  pTP                  VP                  4       '       d   TMVP                  4       4       K<  	  RP                  V4      u #  R # 	  R # )r   z^(.*?)\s+style\s*#\s*\d+r  rN   z\bstyle\s*#\s*\d+\bz^(.*?)\s+style\s*#\s*\d+\b)rR   rP   r  r  r  splitr  isupper
capitalizer  r  )	r  r  htr  r  r4  r   r   m2s	   &&       rB   extract_brandr#    s<   


	"	"	$B
		-rGAggaj CYY[

		1@ !88C= $$&99+TGG8$**,bmm\Bhhqk'')C YY[

		1H )88C=( ' rD   c               $    V ^8  d   QhRRRRRR/# )r<   r  r   r  r?   r@   )rA   s   "rB   rC   rC     s!     	& 	&# 	&c 	&c 	&rD   c                   V'       dY   V P                  4       P                  R V 24      pVR8w  d0   WVR,            p\        VR4      pV'       d   VP                  4       # \        V R4      pV'       d   VP                  4       # R# )z
ABOUT THE i@  r#   r   r  )r  r  r  rR   )r  r  idxchunkvals   &&   rB   extract_about_namer)    sr    oo$$z%%9:"93:.E"5&1Cyy{"
9f
-C399;%2%rD   c                    V ^8  d   QhRRRR/# )r<   rV   r   r?   zDict[str, bool]r@   )rA   s   "rB   rC   rC     s     9 9 9 9rD   c           	     0   \        R  \        P                  ! RV P                  4       4       4       4      pV\        R \        P                  ! RV P                  4       4       4       4      ,          p\	        ^^4       Uu/ uF
  q" R2W!9   bK  	  up# u upi )c              3  8   "   T F  p\        V4      x  K  	  R # 5ir   rv   r  r   s   & rB   r   parse_heights.<locals>.<genexpr>  s     V U1A U   z\b(\d{1,2})\s*-\s*INCH\bc              3  8   "   T F  p\        V4      x  K  	  R # 5ir   r-  r.  s   & rB   r  r/     s     S!RAQ!Rr0  z\b(\d{1,2})\s+INCH\b")r  rP   findallr  r  )rV   foundr  s   &  rB   parse_heightsr5    st    V

+F

 UVVE	SS,CTZZ\!RSSSE+0B<8<acGaj <888s    Bc                    V ^8  d   QhRRRR/# )r<   	brand_strr   r?   zTuple[bool, bool, bool]r@   )rA   s   "rB   rC   rC   $  s     ! !S !-D !rD   c                    T ;'       g    R P                  4       pVP                  R4      pVP                  R4      pVP                  R4      pRV9   d!   VP                  R4      '       g
   RV9   d   RpW#V3# )r   zred wingzirish setterworxzby red wingT)r#  r%  )r7  bis_rwis_isis_worxs   &    rB   classify_brand_familyr>  $  sl    	b!ALL$ELL(Ell6"G!all622!9K  rD   c               $    V ^8  d   QhRRRRRR/# )r<   r   r   max_lenrv   r?   r@   )rA   s   "rB   rC   rC   3  s!      3  s rD   c                   T ;'       g    R P                  4       p\        P                  ! RRV4      pVP                  RR 4      p\        P                  ! RR V\        P                  R7      pVP                  4       P                  RR4      p\        P                  ! RRV4      pV'       g   Rp\        V4      V8  d   VR	V P                  R4      pV# )
r   rM   rN      ®z[^\w\-\.\s]+r  r  z_+bootN)rR   rP   rQ   rO   UNICODEr  r   )r   r@  rJ   s   && rB   _safe_filename_from_namerE  3  s    	A
vsAA			$A
ARZZ8A		#s#A
uc1A
1vhwKs#HrD   c                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   A  s      T rD   c                     \         P                  ! 4       P                  R ,          P                  4       p V P	                  RRR7       V # )ImagesTr   )r   cwdparentresolver  r]  s    rB   _images_dirrL  A  s6    			X	%..0AGGD4G(HrD   c                    V ^8  d   QhRRRR/# )r<   fnamer   r?   r@   )rA   s   "rB   rC   rC   G  s     . .3 .3 .rD   c                F    \        \        R 4      R,          V ,          4      # )z..rH  )r   r   )rN  s   &rB   _rel_image_pathrP  G  s    tDzH$u,--rD   c                  aaa R FH  p V P                  \        P                  V4      pV'       d   VP                  4       '       d   Vu # KH  KJ  	   V P                  \        P
                  R4      pVP                  \        P                  R4      pV'       d   VP                  4       '       d   V#  T P                  \        P                  R4      pT'       g"    T P                  \        P                  R4      pRpRpT EF  p TP                  4       '       g   K  TP                  ;'       g    / p\        TP                  R4      ;'       g    ^ 4      p	\        TP                  R4      ;'       g    ^ 4      p
T	^x8  g   T
^x8  d   K  TP                  R4      ;'       g    R	P                  4       oTP                  R
4      ;'       g    R	P                  4       oTP                  R4      ;'       g    R	P                  4       oY,          p\        ;QJ d    T3R lR 4       F  '       g   K   RM	  RM! T3R lR 4       4      '       d
   TR,          p\        ;QJ d    T3R lR 4       F  '       g   K   RM	  RM! T3R lR 4       4      '       d
   TR,          p\        ;QJ d    T3R lR 4       F  '       g   K   RM	  RM! T3R lR 4       4      '       d
   TR,          pY8  d   TpTpEK  EK  	  T#   \         d     EK	  i ; i  \         d     ELni ; i  \         d    . p ELai ; i  \         d    . p ELLi ; i  \         d     EKY  i ; i)@div.slick-slide.slick-current.slick-active img[itemprop='image']productImageimgmain imgNr   r   heightaltr   classsrcc              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r@   )r  r  rW  s   & rB   r  1_pick_best_product_img_element.<locals>.<genexpr>|  s     P%O8%Or  TFr  c              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r@   )r  r  clss   & rB   r  r[  ~  s     M%L8%Lr  gffffff?c              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r@   )r  r  rY  s   & rB   r  r[    s     R%Q8%Qr  g?)rR  .div.slick-slide.slick-current.slick-active imgHdiv.c-image-carousel__slider-item.js-carousel-item img[itemprop='image'])rC  shoechukkahikermoc)productprimaryheroimage)z	/dw/imagescene7z/imagesstatic)r  r   r  r;  rx   rR  r  r  sizer   rw   r  r#  r  )r{  rG  rT  	containerimgsbest
best_scorerW  szr   hr  rW  r]  rY  s   &           @@@rB   _pick_best_product_img_elementrr  K  s   
	%%boos;Cs''))
 *s''~>	$$R__e<3##%%J
##BOOZ@ 	''U;D DJ	??$$BBbffWo**+AbffX&++!,A3w!c'##E*00b779C##G,2299;C##E*00b779CEEsP%OPsssP%OPPPsM%LMsssM%LMMMsR%QRsssR%QRRR!"
 "' 0 Kc  		    
  	D	6  		s   =L L AL L 1L 4 L%  L9 M M3M#M7MM&(M(M88M3MM%M<MM.MM#M LLL"!L"%L65L69M
	M
MMc                   V ^8  d   QhRR/# r  r@   )rA   s   "rB   rC   rC     s     1 1# 1rD   c                r   R F  p V P                  \        P                  V4      pVP                  R4      ;'       g    RP	                  4       pVP                  R4      ;'       g    RP	                  4       pV'       g	   V'       d   T;'       g    Tu # K  	   V P                  \        P                  R4      p VP                  \        P                  R4      pV'       do   VP                  R4      ;'       g    RP	                  4       pVP                  R4      ;'       g    RP	                  4       pV'       g	   V'       d   T;'       g    T# R F  p T P                  \        P                  T4      pTP                  R4      ;'       g    RP	                  4       pTP                  R4      ;'       g    RP	                  4       pT'       g	   T'       d   T;'       g    Tu # K  	  R#   \
         d     EK  i ; i  \
         d    Rp EL5i ; i  \
         d     Li ; i  \
         d     K  i ; i)	rR  zoomimgr   rY  rS  rT  N)rR  r_  r`  z6div.c-image-carousel__slider-item.js-carousel-item img)z$li[data-orbit-slide="product-1"] imgzli.active imgzul#productImage li.active imgzmain li.active imgrU  )r  r   r  r  rR   rx   rR  )r{  rG  rT  rW  rJ   rl  s   &     rB   _extract_product_image_urlrv    s   	%%boos;C""9-33::<A""5)//R668AAvvA  ''~>		((%@C ""9-33::<A""5)//R668AAvvA
	%%boos;C""9-33::<A""5)//R668AAvvA   G  		  	C	  "  		s   7G/(G/(G/G/	G/G/ H = H H %H =(H &H ?H H H 7H'(H':H'H'H'$H'/G?>G?HH HH H$#H$'H65H6c                    V ^8  d   QhRRRR/# )r<   r  r   r?   r@   )rA   s   "rB   rC   rC     s     8 8c 8c 8rD   c                   T;'       g    RP                  4       pV'       g   R# RV R2p\        4       V,          p \        V 4      pV'       dy   VP                  RR4      pRR/p\        P
                  ! WE^R7      pVP                  '       d9   VP                  '       d'   VP                  VP                  4       \        V4      # R	p \        T 4      pT'       g   R#  T P                  R
T4       \        P                  ! R4        \        T 4        TP!                  \#        T4      4       \        T4      #   \         d     Li ; i  \         d    R	p Li ; i  \         d     Lci ; i  \         d     Lhi ; i  \         d=     T P%                  \#        T4      4       \        T4      u #   \         d      R# i ; ii ; i)zSave the main product image as a temp file (__<style>.png).

Preferred: download the product image URL (zoomimg/src) for best quality.
Fallback: element screenshot if download fails.
Returns relative path like ../Images/__595.png (or empty string).
r   __.pngz&amp;&r.  zMozilla/5.0)r0  r2  NrO  r  )rR   rL  rv  rO   requestsrw   okr  write_bytesrP  rx   rr  r  rr   r  r:  
screenshotr   save_screenshot)r{  r  rN  out_pathimg_urlr0  r8  img_els   &&      rB   capture_product_image_tempr    s    [[b!EtE}u$H
,V4oogs3G#]3GWrBAttt			$$QYY/&u--
 F/7 _agh

3v#h-(u%%5      
    	""3x=1"5)) 			s~   A$D; "%D; E !(E 
E0 $F ;E	E	EEE-,E-0E>=E>G$F41G4G?GGGc               (    V ^8  d   QhRRRRRRRR/# )r<   r  r   	boot_namerel_temp_pathr?   r@   )rA   s   "rB   rC   rC     s(     " "3 "3 "s "s "rD   c                   V'       g   R# \         P                  ! 4       V,          P                  4       pVP                  4       '       g   V# \        P
                  ! RRT ;'       g    RP                  4       4      pV'       g   V# V R2p\        4       V,          p VP                  V4       \        T4      #   \         d    Tu # i ; i)z8Rename ../Images/__<style>.png -> ../Images/<style>.png.r   z[^\dA-Za-z_-]rz  )r   rI  rK  ro   rP   rQ   rR   rL  rO   rx   rP  )r  r  r  temp_absstyle_cleanrN  dest_abss   &&&    rB   finalize_image_filenamer    s    
]*335H??&&)2/B/B/DEKm4 E}u$H" 5!!  s   B: :C
	C
c               $    V ^8  d   QhRRRRRR/# )r<   r  r   
source_urlr?   zTuple[bool, bool]r@   )rA   s   "rB   rC   rC     s"       # 2C rD   c                   T ;'       g    RP                  4       pT;'       g    RP                  4       p\        P                  ! RV4      '       g   RV9   g   RV9   g   RV9   d   R# \        P                  ! RV4      '       g   RV9   g   RV9   g   R	V9   d   R# \	        \        P                  ! R
V4      4      p\	        \        P                  ! RV4      4      pWE3# )zInfer gender flags from URL + header (do NOT use body text; it frequently contains both words).

Priority:
  1) URL slug/query (mens/womens)
  2) Header text (MEN'S / WOMEN'S)
  3) Otherwise: unknown -> (False, False)
r   z/womens(?:[-/]|$)zwomens-zgender=womenzgender=femalez/mens(?:[-/]|$)zmens-z
gender=menzgender=maler  r  )FT)TF)r  r#  rP   r  r>   )r  r  huulmalefemales   &&    rB   infer_genderr    s     ,,B			B


	!	!	#B 
yy%r**i2oSUAUYhlnYn	yy#R((GrM\R=OS`dfSf		-,-D"))OR01F>rD   r  image_rel_pathr  c               8    V ^8  d   QhRRRRRRRRRRRRRRR	R
/# )r<   r  r   r   r  r  r  r  r  r?   r  r@   )rA   s   "rB   rC   rC   $  sb     [ [[
[ [ 	[ [ [ [ [rD   c               H  a< T;'       g    RpT;'       g    RpVP                  4       pVP                  4       p\        W#4      p	\        V	4      '       d   Rp	\	        V	4      w  rpV	'       ga   VP                  4       pVP                  4       pRV9   g   RV9   d   Rp	M!RV9   g   RV9   d   Rp	MRV9   g   RV9   d   Rp	\	        V	4      w  rpVP                  4       p\        W$4      w  ppV'       gw   V'       go   VR,          p\        \        P                  ! R	V4      4      p\        \        P                  ! R
V4      4      pV'       d   V'       g   RpMV'       d   V'       g   RpRV9   pRV9   ;'       g    RV9   pRV9   ;'       g    RV9   ;'       g    RV9   pRV9   ;'       g    T;'       g    T;'       g    TpV'       * pRV9   ;'       g    RV9   pRV9   p\        VR4      pV'       d   VP                  4       P                  4       p\        P                  ! RV4      '       d   RpM\        P                  ! RV4      pV'       d     \        VP                  ^4      4      ^ 8  pMERV9   ;'       g    RV9   pM2\        P                  ! RV4      '       d   RpMRV9   ;'       g    RV9   pRV9   ;'       g    \        P                  ! RV4      RJp RV9   ;'       g    \        P                  ! R V4      RJp!R!V9   ;'       g    R"V9   p"R#V9   ;'       g    \        P                  ! R$V4      RJp#R%V9   ;'       g    R&V9   ;'       d    R'V9   p$R(V9   p%\        VR)4      p&V&'       d    V&P                  4       P                  R*4      MR+V9   p'\        VR,4      p(Rp)V('       dS   V(P                  4       o<\        ;QJ d    V<3R- lRP 4       F  '       g   K   RM	  RM! V<3R- lRP 4       4      '       g   Rp)\        VR.4      ;'       g    \        VR/4      p*Rp+V*'       d/   V*P                  4       P                  4       P                  R*4      p+ML\        P                  ! R0V4      '       g   \        P                  ! R1V4      '       d   R*V9   ;'       g    R2V9   p+T;'       g    RP                  4       p,R3V,9   ;'       g    R4V,9   ;'       g    R5V,9   p-R6V,9   p.R7V,9   p/\!        VR8,           V,           4      p0\        VR94      p1V1P                  4       p2R:V29   ;'       g    R;V29   p3R<V29   ;'       g    R=V29   ;'       g    T3p4\#        4       p5V'       d    \%        VR>4      p6V6P'                  R4       Fz  p7RQ Fq  p8V7P)                  V84      p9\+        V9\,        4      '       g   K,  V9P                  4       '       g   KD  V5P/                  V9P                  4       P                  4       4       Ks  	  K|  	  V''       g-   V5'       d%   V5 F  p:V:R+8X  g   R?V:9   g   K  R@V:9   g   K  Rp' M	  V5'       d[   V5 FT  p:V:P1                  RARB4      p;V3'       g   RCV;9   g   RDV;9   d   Rp3Rp4K1  V4'       d   K;  REV;9   g   RFV;9   g
   RGV;9   g   KR  Rp4KV  	  . \3        V 4      N\3        V4      N\3        V4      N\3        V4      N\3        V	4      N\5        V4      N\5        V4      N\5        V
4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V4      N\5        V 4      N\5        V!4      N\5        V"4      N\5        V#4      N\5        V$4      N\5        V%4      N\5        V'4      N\5        V)4      N\5        V+4      N\5        V-4      N\5        V.4      N\5        V/4      N\5        V0RH,          4      N\5        V0RI,          4      N\5        V0RJ,          4      N\5        V0RK,          4      N\5        V0RL,          4      N\5        V0RM,          4      N\5        V0RN,          4      N\5        V0RO,          4      N\5        V44      N\5        V34      N#   \         d    Rp ELi ; i  \         d     ELi ; i)Ra  
Build a row for the markdown table from the scraped pieces of a product page.

In addition to the previously-supported text parsing, this version accepts the
raw HTML of the page (via the ``html`` parameter) so that additional
features can be inferred from non-visible attributes such as ``alt``,
``title``, or ``aria-label`` on icons.  Passing ``html`` is optional; if
omitted, the function falls back to the older behaviour.
r   zIRISH SETTERr*   WORXr+   zRED WINGr)   :NiX  Nr  r  Tz	STEEL TOEzALUMINUM TOEz	ALLOY TOEzNON-METALLIC TOEzNON METALLIC TOEzCOMPOSITE TOEz
SAFETY TOEzMETATARSAL GUARDz	MET GUARD
WATERPROOFr-   z$\b(non[-\s]?insulated|uninsulated)\bFz(\d+)\s*g\binsulat
thinsulatezSLIP RESISTANTz\bSR\bNzELECTRICAL HAZARDz\bEH\bzPUNCTURE RESISTANTPUNCTUREzSTATIC DISSIPATIVEz\bSD\bzANKLE PROTECTIONankleprotectBOAr.   yeszDEFINED HEELzLeather Typec              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r@   )r  r   lls   & rB   r  '_build_row_from_text.<locals>.<genexpr>  s     `$_q7$_r  r/   
Resolvablez\bresoleable\bz\bresolvable\btrueoxfordathleticra  rb  rc  r   zCountry of Originzmade in usazmade in the usazbuilt in usazassembled in the usar(  DEFINEDHEEL    rN   zMADE IN USAzMADE IN THE USAzBUILT IN USAzASSEMBLED IN USAzASSEMBLED IN THE USAr2   r3   r4   r5   r6   r7   r8   r9   )meshnylonfabrictextilepoly	synthetic)rW  r   r  )r  r#  r#  r  r>  r  r>   rP   r  r  rR   rv   r  rx   r%  r  r5  r  r   r-  rw   r  r   r  rO   rS   rG   )=r  r   r  r  r  r  r  
body_upper
body_lowerr7  r;  r<  r=  r  buheader_upperr  r  first_chunk
male_foundfemale_found	steel_toealuminum_toenon_metal_toe
safety_toesoft_toe	met_guard
waterproofinsulation_lineil	insulatedr  slip_resistantelectrical_hazardpuncture_resistantstatic_dissipativeankle_protectionboadefined_heel_linedefined_heelleather_lineall_leather_upperresoleable_line
resoleable
name_loweroxford_athleticrb  rc  heightsorigin_lineorigin_lowermade_in_usabuilt_in_usa
alt_tokensr0  r  ri  r(  tokrY  r  s=   &&&&$$$                                                     @rB   _build_row_from_textr  $  s   ( RI\\rF"J"Jf0I 	""	1)<E'\\^__R>R#7&Ir\Vr\I2r!1"I 5i @g<<>L3LD&  &"))M;?@
BIIo{CDlD*F z)I!Z/LL;*3LL':5;MQ[;[apt~a~M*,[[[[l[[mJ~H#z1QQ{j7PI+J&y,?O""$**,99<bAAI		."-A% #AGGAJ! 3I '"_EE,"2D	99<jIII"j0QQlj6PI&*4ff"))Iy:Yae:eN,
:ll		)U^@_gk@k.*<[[*PZBZ.*<nn"))IW`BaimBm*j8ppg>S>o>oXaeoXoJC(NCBS$**,77>ZhlvZvL#I~>L!s`$_`sss`$_``` $&y,?llCST]_kClOJ$**,224??F
	$j	1	1RYY?PR\5]5]j(@@Fj,@
**"##%J:-ff:3KffQW[eQeO#Fz!EFTMI56G"9.ABK$$&L L0XX6G<6WK"l2nn8NR^8^nncnL 5J
	 }5D}}T*:D''$-C!#s++		"syy{'8'8':; ; + JCn$c)9fm#	  CFC(AMQ$6:Kq:P"#<^q%8<NRS<SWmqrWr# 1u1t1 	z"1 	~&	1
 	y!1 	D	1 	F1 	E
1 	E
1 	G1 	J1 	I1  	M!1" 	L#1$ 	I%1& 	H'1* 	J+1, 	I-1. 	N/10 	112 	314 	516 	718 	C91< 	L=1> 	?1@ 	JA1D 	OE1F 	FG1H 	E
I1L 	GDMM1N 	GDMO1P 	GDMQ1R 	GDMS1T 	GDMU1V 	GENW1X 	GENY1Z 	GEN[1^ 	L_1` 	Ka1 1o ! % $I%z  		s1   a> A	b )b 5b >bbb! b!c               $    V ^8  d   QhRRRRRR/# )r<   r  r   image_temp_relr?   r  r@   )rA   s   "rB   rC   rC   	  s!     / /3 / /Y /rD   c                j   \         \        V 4      P                  ;'       g    R 9   d   \        R4      hRp\	        V 4       F@  p \        V^R7      p\        WC4      w  rVpTp\        V4      p	\        VVVV	V VVR7      p
V
u # 	  V'       d   Vh\        R4      h  \         d   pTp Rp?Kl  Rp?ii ; i)r   z3HTTP fallback disabled for authenticated order siteNr1  r  r  r  zHTTP fallback failed)
re  r   rd  r>  r  r  r  r  r  rx   )r  r  r  r  r  r  r   r  r  r  r  rb  s   &&          rB   _scrape_product_via_httpr  	  s    Xd^**00b1PQQ(,H4 	q"-D&Ft&O#EF%d+I&-C J !& 
-
..  	H	s   ;BB2%B--B2prefer_httpc               $    V ^8  d   QhRRRRRR/# )r<   r  r   r  r>   r?   r  r@   )rA   s   "rB   rC   rC   	  s&     0M 0M 0Md 0My 0MrD   c          
        R p\        V4      pTp\        4        V'       d    \        WR7      #  \        \        V4      P                  ;'       g    R 9   d   \        V \        4       \        W4       \        V 4       V P                  ;'       g    R p\        Wa4      w  rGp V'       d   \        W4      p\        T ^4      P                  \         P"                  ! \$        P&                  R34      4       T P)                  \$        P&                  R4      P*                  ;'       g    R p	Tp
\-        TTT
T	TTTR7      pT#   \         d     ELi ; i  \         d    R p Li ; i  \         d    \        YR7      u # i ; i)r   )r  r  r  )r  r  r  rx   re  r   rd  rL  r*  r  r:  r+  r  r  r   rS  rT  presence_of_element_locatedr   r  r  rV   r  )r{  r  r  r  r  style_from_urlr  r   r  r  r  r  s   &&$         rB   scrape_productr  	  sa   N"4(ENO	+DPP
"M$..44"5V%89v !!''R"B4"NZ	 !;F!J 	fb!''(F(FU[G\(]^''V<AAGGR	")
 
E  		$  	 N	 $  M'LLMsT   D?  E$ AE$ #E 6A/E$ &E$ ?EEE!E$  E!!E$ $E>=E>c                    V ^8  d   QhRRRR/# )r<   rg   r  r?   rW   r@   )rA   s   "rB   rC   rC   P	  s     6 6 6T 6rD   c           	        \         pR  R lp\        WR7      p. pVP                  RRP                  V4      ,           R,           4       VP                  RRP                  R.\	        V4      ,          4      ,           R,           4       V F  p\	        V4      \	        V4      8  d,   VR.\	        V4      \	        V4      ,
          ,          ,           pM'\	        V4      \	        V4      8  d   VR\	        V4       pVP                  RRP                  V4      ,           R,           4       K  	  \        \        RP                  V4      R,           4       R# )c                   V ^8  d   QhRR/# )r<   r8  r  r@   )rA   s   "rB   rC   $write_markdown.<locals>.__annotate__S	  s      Y rD   c                    V ^ ,          p ^ \        \        P                  ! RRV4      4      3#   \         d    ^T3u # i ; i)r   z\Dr   )rv   rP   rQ   rx   )r8  rJ   s   & rB   	style_key!write_markdown.<locals>.style_keyS	  sE    aD	s266%Q/011 	q6M	s   ". A A )r  rL   z---r   Nr   )
MD_HEADERSr  r  r  r  r`   OUT_MD)rg   r0  r  sorted_rows	out_linesr8  s   &     rB   write_markdownr  P	  s    G -KIS388G,,s23S388UGc'l$:;;cABq6CL RDCL3q6122AVc'l"-3w< Asxx{*S01  9-45rD   c                   V ^8  d   QhRR/# r&  r@   )rA   s   "rB   rC   rC   j	  s      9 rD   c                 d   \         P                  4       '       g   . # . p \         P                  R RR7      P                  4        Fh  pVP	                  4       pV'       g   K  \
        P                  ! RV4      pV'       g   K>  VP                  ^ 4      pW09  g   KW  V P                  V4       Kj  	  V # )rY   rO   )r[   r4  zhttps?://\S+)	r   ro   ru   r  rR   rP   r  r  r  )rj  r   r  r  s       rB   _read_errors_urlsr  j	  s    	D$$gi$HSSUzz|IIot,1
A}A V KrD   c                    V ^8  d   QhRRRR/# )r<   rj  r  r?   rW   r@   )rA   s   "rB   rC   rC   {	  s     6 6y 6T 6rD   c                    V '       g    \         P                  R R7       R# V  Uu. uF  pR\         RV 2NK  	  pp\	        \         RP                  V4      R,           4       R#   \         d     R# i ; iu upi )Tr9  NFAILED x: r   )r   rF  rx   MAX_FAILS_PER_LINKr`   r  )rj  r  r  s   &  rB   _rewrite_errors_filer  {	  sw    	. 	;?@4aw)*#aS14E@TYYu-45	  		 As   A% A7%A43A4c                   V ^8  d   QhRR/# r   r@   )rA   s   "rB   rC   rC   	  s     Z Zd ZrD   c                    \        4       p \        V P                  R . 4      4      p\        V P                  R/ 4      4      p\	        V P                  R. 4      4      p\        V P                  R\        P                  ! 4       4      4      p\	        V P                  R. 4      4      pV'       g   \        4       p\        '       d   \        V 4      pV'       dm   \        V P                  R . 4      4      p\        V P                  R/ 4      4      p\	        V P                  R. 4      4      p\	        V P                  R. 4      4      pRpRp \        V \        RRR7      p\        V4       V P                  R	4      ;'       g    . p	V P                  R
4      ;'       g    / p
\        '       g	   V	'       g   \        ^VR7      pVP                  ^ RR7       \!        V4      w  rV'       d   Tp	M\"        P$                  P'                  R4       V'       d   Tp
 \(        '       d>   V
'       d6   \+        W:4      pV'       d#   W0R&   WR
&   \-        V 4       \/        RV R24       YR	&   \(        '       d   YR
&   \-        T 4       \(        '       dd   V
'       g\    \!        V4      w  rWR
&   \-        V 4        V
'       d6   \+        W:4      pV'       d#   W0R&   WR
&   \-        V 4       \/        RV R24       \2        '       dU   \5        V 4      pV P                  R4      '       g   RV R&   \-        V 4       V'       d   \/        RV R24       \-        V 4       \6        P9                  4       '       dQ   \6        P9                  4       pT	;'       g    .  Uu. uF  p\;        V4      V8X  g   K  VNK  	  p	pWR	&   \-        V 4       \=        V	4      pV^ 8X  d   \?        R4      hV'       d   VPA                  4        \        VVR7      p\=        V4      pVP                  VRR7       \C        V4       UUu/ uF+  w  ppV'       g   K  V^ ,          '       g   K!  V^ ,          VbK-  	  pppV	 EF  pVV9   d   V^,          pVP                  VRR7       K)  Rp  \E        VV4      pV^ ,          '       g   \?        RV 24      hV^,          '       g   \?        RV^ ,           RV R24      h\(        '       dA   V
'       d9   \G        V
P                  V^ ,          R4      4      pV'       d   \I        V4      V^&   \=        V4      ^8  d   V^,          '       g   \I        V4      V^&    \=        V4      ^8  d>   V^,          '       d/   \I        \K        V^ ,          V^,          V^,          4      4      V^&   T^ ,          T9   d   TTTT^ ,          ,          &   M&\=        T4      TT^ ,          &   TPM                  T4       TPO                  T4       \Q        T4      T R &   Y0R&   Y R&   YPR&   \-        T 4       \S        T4       Rp T^,          pTP                  TT'       d   R#MR$R7       EK  	  \S        V4       VP                  VR%R7       \f        '       Ed   V'       Ed   \"        P$                  P'                  R&\=        V4       R'24       . p V'       d    VPe                  4        \        V \        R(R\h        R)7      p\        V4       \C        \	        V4      ^R*7       F  w  ppRp\k        \l        4       F  p \E        VVRR+7      pV'       do   V^ ,          '       d^   V^,          '       dM   V^ ,          V9   d   VVVV^ ,          ,          &   M&\=        V4      VV^ ,          &   VPM                  V4       Rp MK  K  K  	  V'       d5   \"        P$                  P'                  R.V R \=        V4       R/V R"24       K  VPM                  V4       K  	  TpW0R&   WPR&   \-        V 4       \S        V4       \q        V4       \"        P$                  P'                  R1\r         R"24       \t        Pw                  4       '       d(   \"        P$                  P'                  R2\t         R"24        V'       d   VPA                  4         T'       d   TPe                  4        R# R#   \0         d     ELxi ; i  \0         d    T
;'       g    / p
 ELFi ; i  \0         d     ELi ; iu upi u uppi   \0         d     ELi ; i  \0         Ed   p\U        TP                  T^ 4      4      ^,           TT&   \Q        T4      T R &   Y0R&   Y R&   YPR&   \-        T 4       TT,          \V        8  df   TT9  d*   TPM                  T4       \Y        RTT,           RT 24       TPO                  T4       \Q        T4      T R &   YPR&   \-        T 4        Rp?EK  \"        P$                  P'                  RTT,           R \V         R!T R"24       \"        P$                  P'                  RP[                  \\        P^                  ! \a        T4      TTPb                  4      4      R",           4        T'       d   TPe                  4        M  \0         d     Mi ; i\        T \        RRR7      p\        T4        Rp?EKQ  Rp?ii ; i  \0         d     ELi ; i  \0         d@     TP                  R,4       M  \0         d     Mi ; i\        Pn                  ! R-4        EK  i ; i  \0         d   p\"        P$                  P'                  R04       \"        P$                  P'                  RP[                  \\        P^                  ! \a        T4      TTPb                  4      4      R",           4        Rp?ELRp?ii ; i  \0         d     EL6i ; i  \0         d     R# i ; i   T'       d   TPA                  4        M  \0         d     Mi ; i T'       d   TPe                  4        i i   \0         d     i i ; i; i)3rf   rj   rg   rl   rk   Nrf  T)rk  re  rg  re   rh   )r   rl   zRefreshing product links...)r   zN[warn] Link refresh discovered 0 links; falling back to checkpoint link list.
z)Applied preferred International names to z existing rowsri   z
Re-queued z links due to missing URL/Imagez{No product links discovered from catalog page. Login/navigation likely succeeded but product link extraction found nothing.startingz	(resumed)Fz$Style number parsed empty for link: zName parsed empty for style r   )r   r  r  z
Error scraping link (attempt r   z): r   r}  skippeddonez
Starting salvage pass for z failed links...
normal)rk  re  rg  rh  )start)r  r  g      ?z	Salvaged z: z$
Salvage pass encountered an error:
z
DONE. Wrote: z#Some links failed repeatedly; see: )<r|   r  rw   r  r  r   rr   r  AUTO_REPAIR_BAD_ROWSr  r|  RUN_HEADLESSr  REFRESH_PRODUCT_LINKS_EACH_RUNr   r   r  r   r  r   r  r  r   printrx   AUTO_REPAIR_MISSING_MEDIAr  r    rR   r  r  r>  r   r  r  r  rS   r  r  r  r  r  rv   r  r   r  	tracebackformat_exceptiontype__traceback__quitENABLE_SALVAGE_PASSSALVAGE_PAGE_LOAD_TIMEOUTr  SALVAGE_MAX_TRIES_PER_LINKr  r  r  r   ro   )r]  rf   rj   rg   rl   rk   repairedreporterr{  re   rh   fresh_linksfresh_preferred_namesnupdr  rmwantr  r   	processedr  r8  style_to_idxr  r}  r  r  rb  	remainingsalvageds                                 rB   mainr  	  s
   		BrvvlB78J"&rvvmR'@"AK !34DrvvlDIIK89J#'/BB(G#H-/
 -b1RVVL"56JrvvmR89Kvr*+D $RVV,?%D E ,0HFrLWcghF#/552*,&&1B*C*I*Ir))'aJGHOOA$AOB1Fv1N.K +

  e %"7	55/0GD%)6
0?,-'+ I$~^_ #011(7$%B .-o8%:6%B"(7$%#
	"0GD%)6
0?,-'+ I$~^_ %$%b)B66-..*.&'#
2$&EFG# ##%D)6)<)<")<c)<1AWXYAZ^bAbQQ)<Mc"/BM"A:_  NN#%JG
O		
3 -6dOJODAqqQqTT!aOJ!Dz!Q		<BL0(6Cq66*-QRVQW+XYYq66*-I#a&QSTXSYYZ+[\\ :9o2?3F3Fs1vr3RS%3D%9CF 3x!|CFF!/!5As8a<CFF%34KCPQFTWXYTZ\_`a\b4c%dCF
 1v-58\#a&12/24ySV,C(NN4('-j'9B|$!%vJ(3}%.?*+#B'"4(BF NIOOIRTYOHo "t 	tF+ #4#4JJ;C@Q<R;SSefg#%I,j ')'/!&&? $F+(.?)@JGAt$H"#=>,"04"PC"s1vv#a&&#&q6\#9ADDc!f)=$>;>t9LQ$8$(KK$4+/ % 39vs ?$  

((9QCq=N9O8PPRSWRXXZ)[\!((./  K2 %.! vJ&7"#B4  !23

?6("56JJB:,bQR	 	     8"1"7"7R8    d$ K@ % . ! 0(+KOOD!,D(E(IK%'-j'9B|$!%vJ(3}%.?*+#B'"4(,>>'88-44T:-D8I7J#dV.TU"t,+1*+=<(2C./'+JJ$$9+d:K9LAN`Maadeidjjlm JJ$$RWWY-G-GQQRTUTcTc-d%ehl%lm!"KKM$ *2Y`ostF'//?0b % 0  ) ,% &

= 9#, % $% JJsOO,  j

  !IJ

  )C)CDGQPQP_P_)`!adh!hiij$  		
  			  			  		s  5o3 =o3 o3 'o3 /Ao3 o3 c c %5c *o3 o3 c +c4 35c4 (o3 49o3 .3o3 "o3 ?o3 dd!5o3 A	o3  d2dd1o3 7d#;$d# d#(Ad#?d#d-.dBd#&o3 Ao3 .o3 5l9 >k Al9 "$k,k,A
k, l9 2Al9 =A)o3 ''o3 o (o! 0o! co3 co3 c1*c1-o3 0c11o3 4d?o3 do3 d d#d  d##k/B:k)o3 0Bkjkj*'k)j**!ko3 ko3 k)%l9 (k))l9 ,l68l
	l6
l	l6l	l61l9 5l66l9 9oA=oo3 oo3 oo!o0/o03q5pqpqpq p:(p:8q:qqqq__main__>   r   yonr  r  )rP  g?)T)   )P   )x   r   )__conditional_annotations__r   
__future__r   rs   r   r"  rP   r  rG  r   rD  rr   r  r  r|  r  r  html.parserr   r@  pathlibr   typingr   r   r   r   r	   urllib.parser
   r   r   urllib.requestr   r   bs4r   seleniumr   selenium.common.exceptionsr   r   selenium.webdriver.common.byr   selenium.webdriver.common.keysr   "selenium.webdriver.firefox.optionsr   "selenium.webdriver.firefox.servicer   ru  selenium.webdriver.supportr   rT  selenium.webdriver.support.uir   rq   r*  re  r\  r  rw   r   r   rR   r#  r  r  r    __annotations____file__rK  rJ  r  r  rn   r   r  r   r  r  r  r  r  r  PAGE_LOAD_TIMEOUTry  NAV_SETTLE_SECONDSr  rr  r
  r  r  rp   r:   rG   rS   r`   r|   r   r   r   r   r   r  r  r*  r[  rd  r|  r  r  r  r  r  r  r  r  r  r  r$  r6  rL  rX  ra  rm  r  r  r:  r  r  r  r  r  r  r  r  r  r  r  r#  r)  r5  r>  rE  rL  rP  rr  rv  r  r  r  r  r  r  r  r  r  r  r   )r  s   @rB   <module>r3     sp  *X #  	  	   
       "   3 3 4 4 +
   K + / 6 H @ 7 3 #1o $	$  ::>>"46PQ ::>>"46FG zz~~mS1779??AEdd!#0BC!H!N!N!P!V!V!X\{!{  
C >!!#**	.	.<<
77
 
  +  &+ "%'.
   !' 	 	  2	 4B	 DV	 Xb	
 
 
 !1
 3F
 H\  - /D  ( *6    ") 	    ! #( */ 16  "
 J !     #         ::>>"4b9??AIIT <
~I&
4G GZ#L!0AH!
H5 &	5
 5 /5 !%5 5p, %&CN"2$J $4
3#r"JCL=@aH0&REP1DC*H.BL41h"=&@ J"#J-d62	&9!.<@1f8v",*[ [ [ [|/:0MU 0Mb64"6Zz zF rD   