+
    UKia             	      	  a  0 t $ R t^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RIHt ^ RIHtHtHtHtHt ^ RIHtHtH t  ^ RI!H"t"H#t# ^ RI$H%t% ^ R	I&H't' ^ R
I(H)t)H*t* ^ RI+H,t, ^ RI-H.t. ^ RI/H0t0 ^ RI1H2t3 ^ RI4H5t6 ^ RI7H8t8 . ROt9]9^ ,          t:Rt;Rt<]Pz                  P}                  R4      ;'       g    RP                  4       t@]Pz                  P}                  R4      ;'       g    RP                  4       tA]Pz                  P}                  RR4      P                  4       P                  4       R9   tC]Pz                  P}                  RR4      P                  4       P                  4       R9   tDRtER]FR&   ]! ]G4      P                  4       P                  tJ]JR,          tK]JR,          tL]JR,          tM^tNRtOR tPR!tQ]Pz                  P}                  R"R4      P                  4       P                  4       R9   tR]Pz                  P}                  R#]R'       d   R$MR4      P                  4       P                  4       R9   tS. ROtT]U! ]T4      tVRtW^-tX^tYR%tZRt[Rt\Rt]^Zt^^t_^t`]Pz                  P}                  R&R4      P                  4       ;'       g    RtaR' R( ltbR) R* ltcR+ R, ltdR- R. lteR/ R0 ltfR1 R2 ltgR3 R4 lthR5 R6 lti ! R7 R84      tjR9 R: ltkR; R< ltlR= R> ltmR? R@ ltnRA RB ltoRRCRDRERRF]XRGR/RH RI llltpRJ RK ltqRL RM ltr]X]Z^3RN RO lltsRP RQ lttRRR RS lltu ! RT RU]4      tvRV RW ltwRX RY ltxRZ R[ ltyR\ R] ltzR^ t{R_ R` lt|Ra Rb lt}Rc Rd lt~Re Rf ltRg Rh ltRRi Rj lltRk Rl ltRm Rn ltRRo Rp lltRq Rr ltRs Rt ltRu Rv ltRw Rx ltRy Rz ltR{ R| ltR} R~ ltR R ltR R ltR R ltR R ltR R ltR R ltR R ltRR R lltR R ltR R ltR tR R ltR R ltR R ltR R ltR R ltR R ltR R ltRR R lltRRRRRR/R R lltRR R lltRR!/R R lltR R ltR R ltR R ltR R lt]R8X  d
   ]! 4        R# R# )aP  
RW_Site_Scraper_Non_Footwear.py
===============================

Scrapes Red Wing non-footwear catalogs:
- https://order.redwingshoes.com/other-ppe
- https://order.redwingshoes.com/accessories-*

Test mode:
- Set ONLY_STYLE at the top of this file to a style number (e.g. "400")
  to scrape just that single product.

Outputs (next to this script):
- RW_Non_Footwear_Site_Scrape.md
- RW_Site_Scraper_Non_Footwear_checkpoint.json   (resume state)
- RW_Site_Scraper_Non_Footwear_errors.txt        (links that failed repeatedly)

Stability features:
- `safe_get()` uses short timeouts + window.stop() so Selenium doesn't hang forever
- HTTP fallback (no Selenium) for the few product pages that still time out
- Optional salvage pass at the end to retry hard failures (HTTP-first)

Cross-platform (Windows + Linux Cinnamon):
- Headless Firefox (default)
- Geckodriver resolution "like Parts_Auto" (explicit Service path; no Selenium Manager):
    1) GECKODRIVER_PATH env var (file or directory)
    2) geckodriver(.exe) on PATH
    3) auto-download geckodriver (GitHub releases) into a user cache dir
    4) (optional) if double-click/no terminal and Tk is available, prompt to pick geckodriver

Feature columns are 1/0 (not Yes/No).
Includes Brand (string) + brand family flags (Red Wing / Irish Setter / Worx).

Dependencies:
- Python 3.9+
- Firefox installed
- Selenium installed:
    Linux Mint/Ubuntu: sudo apt install -y python3-selenium
    Windows: python -m pip install selenium

Notes for Linux Mint PEP 668:
- Prefer `python3-selenium` from apt (as above).
- This script does NOT require webdriver-manager.
)annotationsN)
HTMLParser)Path)DictListOptionalSetTuple)parse_qsurljoinurlparse)Requesturlopen)BeautifulSoup)	webdriver)TimeoutExceptionWebDriverException)By)Keys)Options)Service)expected_conditions)WebDriverWaitzorder.redwingshoes.comRW_SITE_USERNAMEzrwss614@redwingshoes.comRW_SITE_PASSWORDzWelcomeBack99!RW_HEADLESS1RW_REFRESH_LINKS str
ONLY_STYLEzRW_Non_Footwear_Site_Scrape.mdz,RW_Site_Scraper_Non_Footwear_checkpoint.jsonz'RW_Site_Scraper_Non_Footwear_errors.txtTRED WING FOR BUSINESSFRW_CAPTURE_IMAGESRW_BLOCK_IMAGES0g      ?GECKODRIVER_PATHc                    V ^8  d   QhRRRR/# )   vboolreturnr    )formats   "EPulled_Info\Boot_Features\RW_Scrapers\RW_Site_Scraper_Non_Footwear.py__annotate__r.      s      4 C     c                    V '       d   R # R# )r   r$   r+   )r(   s   &r-   b01r1      s    3r/   c                    V ^8  d   QhRRRR/# r'   sr   r*   r+   )r,   s   "r-   r.   r.      s      c c r/   c                    T ;'       g    R P                  RR4      p \        P                  ! RRV 4      P                  4       p V # )r   |z\|\s+ )replaceresubstrip)r4   s   &r-   md_escape_cellr=      s:    	
b#u%A
vsA$$&AHr/   c               $    V ^8  d   QhRRRRRR/# )r'   pathr   textr   r*   Noner+   )r,   s   "r-   r.   r.      s!      t 3 4 r/   c                    V P                  V P                  R ,           4      pVP                  VRR7       VP                  V 4       R# )z.tmputf-8encodingN)with_suffixsuffix
write_textr9   )r?   r@   tmps   && r-   atomic_writerJ      s7    


4;;/
0CNN4'N*KKr/   c                   V ^8  d   QhRR/# )r'   r*   r   r+   )r,   s   "r-   r.   r.      s     <
 <
 <
r/   c                    \         P                  4       '       g4   R \        R\        R. R. R. R/ RRR/ R	. R
\        P                  ! 4       RR/#  \
        P                  ! \         P                  RR7      4      p \        V P                  R ^ 4      4      pV\        8w  d[    \         P                  \         P                  RV R24      4       R \        R\        R. R. R. R/ R
\        P                  ! 4       RR/# V P                  R \        4       V P                  R\        4       V P                  R. 4       V P                  R. 4       V P                  R. 4       V P                  R/ 4       V P                  R	. 4       V P                  R
\        P                  ! 4       4       V P                  RR4       V #   \         d     Li ; i  \         dr     \         P                  \         P                  R4      4       M  \         d     Mi ; iR \        R\        R. R. R. R/ RRR/ R	. R
\        P                  ! 4       RR/u # i ; i)versioncatalog_urlsproduct_links
done_linksrowspreferred_namesmedia_repair_doneFfail_countshard_failed_links
started_atgeckodriver_pathNrC   rD   z.json.vz.bakz.json.corrupt)
CHECKPOINTexistsCHECKPOINT_VERSIONCATALOG_URLStimejsonloads	read_textintgetr9   rF   	Exception
setdefault)dataold_vers     r-   load_checkpointrf      s&   )LR"Br2$))+
 	
-
zz*...@Adhhy!,-(("":#9#9GG9D:Q#RS -brdiik"D	 	 		#565,b)#r*+R0diik2*D1+  ,  
	z55oFG 		 )LR"Br2$))+
 	

sb   AG	  ,F8 ,G	 9B>G	 8GG	 GG	 	I(G>=I>H	IH6IIc                    V ^8  d   QhRRRR/# )r'   rd   r   r*   rA   r+   )r,   s   "r-   r.   r.     s     I I$ I4 Ir/   c           	     T    \        \        \        P                  ! V ^RR7      4       R# )r'   T)indent	sort_keysN)rJ   rX   r]   dumps)rd   s   &r-   save_checkpointrl     s    TZZQ$GHr/   c                    V ^8  d   QhRRRR/# )r'   liner   r*   rA   r+   )r,   s   "r-   r.   r.     s     & &C &D &r/   c                    \        \        R RR7      ;_uu_ 4       pVP                  V P                  4       R,           4       RRR4       R#   + '       g   i     R# ; i)arC   rD   
N)open
ERRORS_TXTwriterstrip)rn   fs   & r-   write_errors_linerw     s9    	j#	0	0A	$% 
1	0	0	0s   'AA	c                   V ^8  d   QhRR/# r'   r*   r)   r+   )r,   s   "r-   r.   r.     s      T r/   c                 4     ^ RI p R#   \         d     R# i ; i)    NTF)tkinterrb   )r|   s    r-   _can_use_tkr}     s     s    c                    V ^8  d   QhRRRR/# )r'   titler   r*   Optional[str]r+   )r,   s   "r-   r.   r.     s        r/   c                   \         P                  P                  4       '       g   \        4       '       g   R #  ^ R Ip^ RIHpHp VP                  4       pVP                  4        VP                  RR4       VP                  RR4       VP                  V R7      pVP                  4        T;'       g    RP                  4       pV'       d   V# R #   \         d     R # i ; i)N)
filedialog
messageboxz-topmostTRW Site ScraperzCould not find geckodriver automatically.

Please select the geckodriver executable.
Windows: geckodriver.exe
Linux: geckodriver)r   r   )sysstdoutisattyr}   r|   r   r   Tkwithdraw
attributesshowinfoaskopenfilenamedestroyr<   rb   )r   tkr   r   rootr?   s   &     r-   _tk_pick_filer     s    
zz+--2uuw
D);	
 )))6

!!#t%% s$   A:C 3C C C CCc                  B    ] tR tRtRtR R ltRR R lltR R ltR	tR
# )ProgressReporteri2  zDTTY progress bar, or Tk window if launched by double-click (no TTY).c                    V ^8  d   QhRRRR/# )r'   totalr`   rV   floatr+   )r,   s   "r-   r.   ProgressReporter.__annotate__5  s     ! !c !u !r/   c                	   \        \        V4      ^4      V n        W n        \        P
                  P                  4       V n        RV n        RV n	        RV n
        RV n        V P                  '       EgN   \        4       '       Ed;    ^ RIp^ RIHp VP                  4       V n	        V P                  P!                  R4       V P                  P#                  R4       V P                  P%                  RR4       VP'                  V P                  RRR7      V n
        V P                  P)                  R	^RR
7       VP+                  V P                  V P                  RR7      V n        V P                  P)                  ^RR7       RV n        V P                  P-                  4        V P                  P/                  4        R# R# R#   \0         d    RT n         R# i ; i)   FN)ttkr   620x150zStarting...w)r@   anchorx)fillpadxpadyiD  )maximumlength)r   r   T)      )r{   
   )maxr`   r   rV   r   r   r   use_ttygui_root_label_pbarr}   r|   r   r   r   geometry	resizableLabelpackProgressbarupdate_idletasksupdaterb   )selfr   rV   r   r   s   &&&  r-   __init__ProgressReporter.__init__5  sU   UQ'
$zz((*

+--!$'UUW


  !23

##I.

$$UE2 hhtzzchR  c A __TZZTW_X


Rg6

++-

!!#% #0&  ! !s   D7G GGc               $    V ^8  d   QhRRRRRR/# )r'   currentr`   noter   r*   rA   r+   )r,   s   "r-   r.   r   T  s!      c  d r/   c                	   \        ^ \        \        V4      V P                  4      4      pV P                  '       d   V P
                  '       d   V RV P                   RV 2P                  4       pV P                  '       d   V P                  P                  VR7       V P                  '       d   WP                  R&   V P
                  P                  4        V P
                  P                  4        R# ^ pWP                  ,          p\        \        WT,          4      4      pRV,          RWF,
          ,          ,           p\        \        P                  ! 4       V P                  ,
          R4      pW,          p	V	R	8  d   V P                  V,
          V	,          MR
p
RV RV RV P                   RV^d,          R R\        V
4       R2pV'       d   VRV 2,          p\        P                   P#                  RVR,          ,           4       \        P                   P%                  4        WP                  8X  d@   \        P                   P#                  R4       \        P                   P%                  4        R# R# )r{   /z  )r@   valueN#-g-C6?g&.>        [z]  (z5.1fz%) ETA r4   :N   Nrq   )r   minr`   r   r   r   r<   r   configr   r   r   roundr\   rV   r   r   rt   flush)r   r   r   msgwidthfracfilledbarelapsedrateetas   &&&        r-   r   ProgressReporter.updateT  s   aS\4::67888


IQtzzl"TF399;C{{{""",zzz&-

7#JJ'')JJ#U4<()FlSEN33diikDOO3V< /3d{tzzG#t+#b	4::,bc$ws3xjPQRRv;C

D	)*

jj JJT"JJ !r/   c                   V ^8  d   QhRR/# r'   r*   rA   r+   )r,   s   "r-   r.   r   t  s      t r/   c                	    V P                   '       d2   V P                  '       d    V P                  P                  4        R # R # R #   \         d     R # i ; iN)r   r   r   rb   )r   s   &r-   closeProgressReporter.closet  sC    888




""$ #8  s   A AA)r   r   r   r   rV   r   r   Nr   )	__name__
__module____qualname____firstlineno____doc__r   r   r   __static_attributes__r+   r/   r-   r   r   2  s    N!>@ r/   r   c                   V ^8  d   QhRR/# r'   r*   r   r+   )r,   s   "r-   r.   r.     s      D r/   c                    \         P                  R 8X  d   \         P                  P                  R4      ;'       gF    \         P                  P                  R4      ;'       g    \	        \
        P                  ! 4       4      p \        V 4      R,          pM^\         P                  P                  R4      ;'       g&    \	        \
        P                  ! 4       R,          4      p \        V 4      R,          pVP                  RRR7       V# )	ntLOCALAPPDATAAPPDATARW_Site_ScraperXDG_CACHE_HOMEz.cacherw_site_scraperTparentsexist_ok)osnameenvironra   r   r   homemkdir)baseds     r-   
_cache_dirr     s    	ww$zz~~n-^^	1J^^cRVR[R[R]N^J**zz~~./NN3tyy{X7M3NJ**GGD4G(Hr/   c                   V ^8  d   QhRR/# )r'   r*   r   r+   )r,   s   "r-   r.   r.     s     # #= #r/   c                    \         '       d{   \        \         4      p V P                  4       '       d   \        V 4      # V P	                  4       '       d5   R F.  pW,          pVP                  4       '       g   K#  \        V4      u # 	  R F3  p\
        V,          pVP                  4       '       g   K(  \        V4      u # 	  R F7  p\        4       V,          pVP                  4       '       g   K,  \        V4      u # 	  \        P                  R8w  ds   \        P                  ! 4       R,          \        P                  ! 4       R,          R,          3 F/  pVR,          pVP                  4       '       g   K$  \        V4      u # 	  \        P                  ! R4      ;'       g    \        P                  ! R 4      p V '       d   V # R# )geckodriver.exegeckodriverr   binz.localN)r   r   )r%   r   is_filer   is_dirBASE_DIRr   r   r   r   shutilwhich)pr   candr   s       r-   _resolve_from_env_or_pathr     s3   !"99;;q6M88:::x<<>>t9$ ; 3$<<>>t9 3 3|d"<<>>t9 3 
ww$))+%tyy{X'='EFA}$D||~~4y  G 	]#FFv||4E'FAr/   c                   V ^8  d   QhRR/# )r'   r*   zTuple[str, str]r+   )r,   s   "r-   r.   r.     s     ! !_ !r/   c                    \         P                  P                  4       p \        P                  ! 4       P                  4       pV P	                  R4      '       d.   R\        P
                  ! 4       ^ ,          9   pV'       d   R	# RR3# V P	                  R4      '       d   RV9   g   RV9   d   R
# R# V R8X  d   RV9   g   RV9   d   R# R# R# )zG
Returns (asset_contains, archive_type) matching geckodriver releases.
win64win32ziplinuxaarch64arm64darwin)win64r  )zlinux-aarch64tar.gz)linux64r	  )zmacos-aarch64r	  )macosr	  )r   platformlowermachine
startswitharchitecture)sysplatmachis_64s      r-   _platform_asset_keyr    s     ll  "G##%D%  --/22 55gu55'""4..$$(d?i4/..""  r/   c                    V ^8  d   QhRRRR/# )r'   dest_dirr   r*   r+   )r,   s   "r-   r.   r.     s     A A4 AD Ar/   c           	         \        4       w  rRp\        VRR/R7      p\        V^R7      ;_uu_ 4       p\        P                  ! VP                  4       P                  RRR7      4      pR	R	R	4       XP                  R
. 4      pR	pR	p	V FI  p
V
P                  RR4      pW9   g   K  VP                  V4      '       g   K6  V
P                  R4      pTp	 M	  V'       g   \        RV RV R24      hW	,          p\        VRR/R7      p\        V^<R7      ;_uu_ 4       p\        VR4      ;_uu_ 4       p\        P                  ! W^4       R	R	R	4       R	R	R	4       \        P                  R8X  d   RMRpW,          pVR8X  d   \        P                   ! VR4      ;_uu_ 4       pVP#                  4        FI  pVP                  V4      '       g   K  VP%                  VV R7       V V,          pVP'                  V4        M	  R	R	R	4       M\(        P                  ! VR4      ;_uu_ 4       pVP+                  4        Fu  pVP                  P                  RV,           4      '       g   VP                  V8X  g   K>  VP%                  VV R7       V VP                  ,          pVP'                  V4        M	  R	R	R	4        VP-                  RR7       \        P                  R8w  dx    \        P0                  ! V4      p\        P2                  ! VVP4                  \0        P6                  ,          \0        P8                  ,          \0        P:                  ,          4       VP=                  4       '       g   \        R4      hV#   + '       g   i     EL; i  + '       g   i     ELN; i  + '       g   i     ELZ; i  + '       g   i     EL; i  + '       g   i     EL#; i  \.         d     EL!i ; i  \.         d     Li ; i)z\
Downloads and extracts latest geckodriver into dest_dir.
Returns path to extracted driver.
z@https://api.github.com/repos/mozilla/geckodriver/releases/latest
User-AgentzRW_Site_Scraper/1.0headerstimeoutrC   r9   errorsNassetsr   r   browser_download_urlz'Could not find a geckodriver asset for r   z).wbr   r   r   r  r)r?   zr:gzr   T
missing_okz5Download succeeded but geckodriver was not extracted.)r  r   r   r]   r^   readdecodera   endswithRuntimeErrorrr   r   copyfileobjr   r   zipfileZipFilenamelistextractr9   tarfile
getmembersunlinkrb   statchmodst_modeS_IXUSRS_IXGRPS_IXOTHrY   )r  	asset_keyarchive_typeapireqr"  rd   r  dl_urldl_namerp   r   archive_pathreq2rv   driver_nameextracted_pathzmemberr   tsts   &                     r-   _download_latest_geckodriverrE    s   
 23I
LC
#.CD
EC	b	!	!Qzz!&&(//')/DE 
" XXh#FFGuuVR |!<!<UU12FG  DYKrR^Q__abcc%L6L2G#HID	r	"	"alD)A)AQ1  *B	" (*ww$#MK+Nu__\3//1**,??;//IIf8I4 6)AIIn- ' 0/ \\,//1,,.;;''k(9::fkk[>XIIf8I4 6;;.AIIn- ) 0t, 
ww$	(BHH^RZZ$,,%>%MPTP\P\%\]   ""RSSq 
"	!	!$ *B)A)A	"	"	" 0// 0//    		sy   6M9N!1N	N!(N51N5%A
O	4;O	8O  A6O/ 9N
	N	N!!N2	5O		O	O,+O,/O=<O=c                    V ^8  d   QhRRRR/# )r'   ckr   r*   r   r+   )r,   s   "r-   r.   r.     s     !
 !
4 !
C !
r/   c                R   \        4       pV'       d"   \        V4      P                  4       '       d   V# V P                  R 4      ;'       g    RP	                  4       pV'       d"   \        V4      P                  4       '       d   V#  \        4       p\        V4      p\        V4      V R &   \        V 4       \        V4      #   \         d\   p\        R4      pT'       d6   \        T4      P                  4       '       d   Y`R &   \        T 4       Tu Rp?# \        RT 24      hRp?ii ; i)rW   r   z$Select geckodriver / geckodriver.exeNzUnable to locate or install geckodriver.

Fix options:
  - Put geckodriver on PATH
  - OR set GECKODRIVER_PATH to the full driver path
  - OR install via package manager (Linux often: sudo apt install firefox-geckodriver)

Underlying error: )r   r   rY   ra   r<   r   rE  r   rl   rb   r   r(  )rG  r   saveddestdriver_pathepickeds   &      r-   ensure_geckodriverrN    s    !#AT!W^^ VV&'--2446Ee##%%
|248!$[!1; 
EFd6l))++%+!"BM!
 "#%
 	

s%   8C   D&AD!D&D!!D&page_load_strategyeagerblock_imagespage_load_timeout
user_agentc               4    V ^8  d   QhRRRRRRRRRR	R
RRR/# )r'   rG  r   headlessr)   rO  r   rQ  rR  r`   rS  r   r*   zwebdriver.Firefoxr+   )r,   s   "r-   r.   r.   0  sN     5 555 	5
 5 5 5 5r/   c               z   \        V 4      p\        4       pV'       d   VP                  R4       \        '       d   VP                  R4       V'       d   VP	                  RV4       VP                  RR4       VP                  RR4       VP                  RR	4       VP                  R
R4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR4       VP                  RR	4       VP                  RR	4       V'       d   VP                  R^4       V'       d   VP                  RV4       \        VR7      p\        P                  ! WR7      p	V	P                  V4       V	P                  \        4       V	# )a6  Start Firefox using an explicit geckodriver path (no Selenium Manager).

Args:
    page_load_strategy: "eager" returns after DOMContentLoaded. "normal" waits for full load.
    block_images: If True, blocks images to reduce load stalls/timeouts.
    page_load_timeout: Seconds for Selenium navigation timeout.
z	-headlessz-privatepageLoadStrategyzdom.webnotifications.enabledFzmedia.volume_scalez0.0z!browser.privatebrowsing.autostartTznetwork.http.http3.enablezbrowser.cache.disk.enablezbrowser.cache.memory.enablezbrowser.cache.offline.enableznetwork.http.use-cachezplaces.history.enabledzsignon.rememberSignonsz"privacy.trackingprotection.enabledz)privacy.trackingprotection.pbmode.enabledzpermissions.default.imagezgeneral.useragent.override)executable_path)serviceoptions)rN  r   add_argumentPRIVATE_BROWSER_MODEset_capabilityset_preferenceFirefoxServicer   Firefoxset_page_load_timeoutset_script_timeoutSCRIPT_TIMEOUT)
rG  rU  rO  rQ  rR  rS  geckorZ  rY  drivers
   &&$$$$    r-   create_driverrf  0  s     r"EiG[)Z(13EF 95A/7>E6>6>8%@95A3U;3U;3U;?FFM:A>;ZHU3Gw@F
  !23
n-Mr/   c                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.   h  s      4 r/   c                     V P                  4         V P                  R4       V P                  R4       R#   \         d     L2i ; i  \         d     R# i ; i)z;Clear cookies/storage for a clean private session baseline.about:blankz}
            try { localStorage.clear(); } catch (e) {}
            try { sessionStorage.clear(); } catch (e) {}
            N)delete_all_cookiesrb   ra   execute_script)re  s   &r-   reset_browser_staterl  h  s^    !!#	

=!	
	    s    7 "A AAAAc                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.   ~  s      t r/   c                     \         w  r \        P                  ! \        P                  ! W4      4       R #   \
         d    \        P                  ! T 4        R # i ; ir   )REQUEST_DELAY_RANGEr\   sleeprandomuniformrb   )lohis     r-   _polite_delayru  ~  s=     FB

6>>")* 

2s   *6 !AAc          
     ,    V ^8  d   QhRRRRRRRRRR	/# )
r'   urlr   r  r`   settler   	max_triesr*   rA   r+   )r,   s   "r-   r.   r.     sC     C C	C C 	C
 C 
Cr/   c                   Rp\        ^V^,           4       EF  p V P                  V4       \        4         \        V RR4      pVe   \	        VR4      '       d>   \	        VP
                  R4      '       d"   \        V4      ^
,           VP
                  n        \	        VR4      '       d>   \	        VP                  R4      '       d"   \        V4      ^
,           VP                  n        T P                  T4       \        P                  ! T4        T P                  \        P                  R4        R# 	  V'       d   VhR#   \         d     Lfi ; i  \         d     L/i ; i  \          d   pTp T P#                  R4       M  \         d     Mi ; i\        P                  ! T4        T P                  \        P                  R4        Rp? R#   \         d      Rp?M!i ; iRp?i\$         d   pTp Rp?MRp?ii ; i T P                  R4       M  \         d     Mi ; i\        P                  ! R	T,          4       EK0  )
zNavigate without getting stuck on pages that never fully finish loading.

- Uses page_load_timeout
- On timeout, calls window.stop() and continues if the DOM exists
- Retries a couple times with light cleanup
Ncommand_executor_client_configr  _connbodyzwindow.stop();ri  g      ?)rangera  ru  getattrhasattrr|  r`   r  r}  rb   ra   r\   rp  find_elementr   TAG_NAMEr   rk  r   )	re  rw  r  rx  ry  last_excattemptcerL  s	   &&&&&    r-   safe_getr    s    )-HIM*)	((1O
V%7>>r#344ARART]9^9^47L24E))1r7++)0L0L+.w<"+<( JJsO JJv##BKK8 5 +f  K       	H%%&67 JJv##BKK8  " 	H		JJ}% 		

4'>"s   E!B/D?&'E! E?E
E!EE!EE!EE!!G?,G%/F G%FG%FG%) GG"G%!G""G%%G?2G?3G::G?HH#"H#c                    V ^8  d   QhRRRR/# )r'   rw  r   r*   	List[str]r+   )r,   s   "r-   r.   r.     s      s y r/   c                
   . pV 3 F  pW!9  g   K  VP                  V4       K  	  RV 9   d-   V P                  RR^4      pW19  d   VP                  V4       V# V P                  RR^4      pW19  d   VP                  V4       V# )zOReturn a small set of URL variants (www/non-www) to dodge occasional redirects.z//www.z//)appendr9   )rw  outsuu2s   &   r-   _url_variantsr    s~    DU=KKN  3[[4+>KKO
 K [[x+>KKOKr/   c               $    V ^8  d   QhRRRRRR/# )r'   rw  r   r  r`   r*   r+   )r,   s   "r-   r.   r.     s!     2 2S 23 2 2r/   c           	         \        V R RRRRR/R7      p\        W!R7      ;_uu_ 4       pVP                  4       pRRR4       XP                  R	R
R7      #   + '       g   i     L#; i)r  z_Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36Acceptz?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zAccept-Languagezen-US,en;q=0.9r  r  NrC   r9   r  )r   r   r%  r&  )rw  r  r:  r"  rd   s   &&   r-   _fetch_htmlr    sc    
{W/
C 
	&	&!vvx 
' ;;wy;11 
'	&s   AA$	c                  F   a  ] tR tRtRt0 RmtV 3R ltR tR tR t	Rt
V ;t# )	_VisibleTextExtractori  z;Very small HTML->visible text extractor (no external deps).c                	2   < \         SV `  4        . V n        R # r   )superr   parts)r   	__class__s   &r-   r   _VisibleTextExtractor.__init__  s     "
r/   c                	~    VP                  4       V P                  9   d   V P                  P                  R 4       R# R# rq   Nr  _BLOCK_TAGSr  r  )r   tagattrss   &&&r-   handle_starttag%_VisibleTextExtractor.handle_starttag  -    99;$***JJd# +r/   c                	~    VP                  4       V P                  9   d   V P                  P                  R 4       R# R# r  r  )r   r  s   &&r-   handle_endtag#_VisibleTextExtractor.handle_endtag  r  r/   c                	    V'       d6   VP                  4       '       d   V P                  P                  V4       R # R # R # r   )r<   r  r  )r   rd   s   &&r-   handle_data!_VisibleTextExtractor.handle_data  s(    DJJLLJJd# !4r/   )r  >   r   brh1h2h3h4h5h6litdthtrdivfooterheaderarticlesection)r   r   r   r   r   r  r   r  r  r  r   __classcell__)r  s   @r-   r  r    s&    EK#$$$ $r/   r  c                    V ^8  d   QhRRRR/# r'   htmlr   r*   r+   )r,   s   "r-   r.   r.     s     
 
 
 
r/   c                >   \        4       p VP                  V 4       R P                  VP                  4      p\
        P                  ! V4      p\        P                  ! RRV4      p\        P                  ! RRV4      pVP                  4       #   \         d     L~i ; i)r   z\n{3,}z

z[\t\r]+r8   )
r  feedrb   joinr  html_libunescaper:   r;   r<   )r  r   raws   &  r-   _html_to_textr    s    A	t ''!''
C


C
 C
&&FC
(C
&&S#
&C99;  s   B BBc                    V ^8  d   QhRRRR/# r  r+   )r,   s   "r-   r.   r.     s      C C r/   c                \   \         P                  ! R V \         P                  \         P                  ,          R7      pV'       g   R# VP	                  ^4      p\         P
                  ! RRV4      p\        P                  ! V4      p\         P
                  ! RRV4      P                  4       pV# )z<h1\b[^>]*>(.*?)</h1>flagsr   <[^>]+>r8   r7   )	r:   search
IGNORECASEDOTALLgroupr;   r  r  r<   )r  minners   &  r-   _extract_first_h1r    sw    
		*D		8QRAGGAJEFF:sE*Ee$EFF63&,,.ELr/   c                    V ^8  d   QhRRRR/# r  r+   )r,   s   "r-   r.   r.   !  s     3# 3# 3# 3#r/   c                  aa R0o0 R	moR R lpR VV3R llp\         P                  ! RV \         P                  \         P                  ,          R7      pV'       d(   V! VP	                  ^4      4      pV! V4      '       d   V# \         P
                  ! RV \         P                  \         P                  ,          R7       F,  pV! VP	                  ^4      4      pV! V4      '       g   K*  Vu # 	  \        V 4      # )
zExtract a likely product title from HTML.

Many Red Wing product pages place the actual product name in <h3>, while <h1>
can be a site/banner header (e.g., 'RED WING FOR BUSINESS'). We therefore
try meaningful <h3> first, then fall back to <h1>.
r!   c                    V ^8  d   QhRRRR/# )r'   r  r   r*   r+   )r,   s   "r-   r.   ,_extract_first_heading.<locals>.__annotate__.  s      c c r/   c                    \         P                  ! R RV 4      p \        P                  ! V 4      p \         P                  ! RRV 4      P	                  4       p V # )r  r8   r7   )r:   r;   r  r  r<   )r  s   &r-   _clean&_extract_first_heading.<locals>._clean.  sD    z3.!!%(vsE*002r/   c                    V ^8  d   QhRRRR/# r'   r4   r   r*   r)   r+   )r,   s   "r-   r.   r  4  s      s t r/   c                   < T ;'       g    R P                  4       p V '       g   R# V P                  4       pVS9   g   VS9   d   R# \        V4      ^8:  d   R# \        P                  ! RV4      '       g   R# R# )r   Fz[A-Z]T)r<   upperlenr:   r  )r4   upBADSTOPs   & r-   _ok#_extract_first_heading.<locals>._ok4  s[    WW"OOWWY9d
r7a<yy2&&r/   z<td[^>]*class=['\"][^'\"]*prTitle[^'\"]*['\"][^>]*>\s*Name\s*</td>\s*<td[^>]*class=['\"][^'\"]*prValue[^'\"]*['\"][^>]*>(.*?)</td>r  z<h3\b[^>]*>(.*?)</h3>>	   CARESIZINGDETAILSREVIEWSFEATURES
TECHNOLOGYSPECIFICATIONS
SIZE & FITRELATED PRODUCTS)r:   r  r  r  r  finditerr  )r  r  r  mnamer4   mh3r  r  s   &     @@r-   _extract_first_headingr  !  s     #
#CD
  II	Immbii'	E 5;;q>"q66H {{3TQSQZQZAZ[399Q< q66H \
 T""r/   c                    V ^8  d   QhRRRR/# )r'   hrefr   r*   r)   r+   )r,   s   "r-   r.   r.   Z  s     " "3 "4 "r/   c                   a T ;'       g    R P                  4       p V '       g   R# V P                  4       pVP                  R4      '       g/   VP                  R4      '       g   VP                  R4      '       d   R# RV9   d   R#  \        V 4      pVP                  ;'       g    R P                  4       P                  R4      o\        VP                  ;'       g    R 4      p\        ;QJ d*    V3R lR\        ,            4       F  '       g   K   RM	  RM! V3R lR\        ,            4       4      '       dj   \        P                  ! R	S4      '       dK   R FB  pVP                  V4      '       g(   VP                  VP                  4       4      '       g   KA   R# 	  R# R# R
S9   d    \        P                  ! RS4      '       d   R# \        P                  ! RS4      '       d   R# R#   \         d	    To/ p ELi ; i)r   Fjavascript:zmailto:ztel:z/safety-boot/Tr   c              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r+   ).0hintr?   s   & r-   	<genexpr>#_is_product_link.<locals>.<genexpr>k  s     
N$MD4<$M   z1.*/(?:footwear-[^/]+|other-ppe|accessories-[^/]+)z	/product/z\d{3,6}z/\d{3,6}\.html$)z
/footwear-stylestyleNumberskuitempid	productId)r<   r  r  r   r?   ru   r
   queryrb   anyNON_FOOTWEAR_PATH_HINTSr:   	fullmatchra   r  )r  lowparsedr   keyr?   s   &    @r-   _is_product_linkr  Z  su   JJBD
**,C
~~m$$y(A(AS^^TZE[E[#$!!r((*11#6++,
 s
NO6M$M
Nsss
NO6M$M
NNN<<LdSSR99S>>UYYsyy{%;%; S  dryyT:: 
yy#T**+  s   G* %7G* G* *G=<G=c                   . p\        4       p V P                  \        P                  R4      pV Fr  p VP	                  R4      ;'       g    RP                  4       pV'       d   WR9   d   K=  \        V4      '       g   KP  VP                  V4       VP                  V4       Kt  	   V P                  \        P                  R4      pV Fr  p VP	                  R4      ;'       g    RP                  4       pV'       d   WR9   d   K=  \        V4      '       g   KP  VP                  V4       VP                  V4       Kt  	  V'       g    V P                  \        P                  R4       Fr  p VP	                  R4      ;'       g    RP                  4       pV'       d   WR9   d   K=  \        V4      '       g   KP  VP                  V4       VP                  V4       Kt  	  V# V#   \         d    Rp ELi ; i  \         d     EL^i ; i  \         d    Rp EL"i ; i  \         d     Li ; i  \         d    Rp Li ; i  \         d     T# i ; i)u  Locate product anchors on the catalog page.

The Red Wing wholesale catalog renders product tiles with specific CSS
classes.  Each tile (<li>) contains two links pointing at the same
product detail page: one wrapping the image and one wrapping the product
name.  Both have an href that ends in ".html" with the numeric style
number.  On the current site the anchor for the name has the classes
``c-product-tile__pdp-link js-product-name``, while the image link has
no special class but is still within the tile.  This helper first
attempts to extract anchors using those specific selectors and falls
back to scanning all anchors when necessary.

Returns a list of WebElement objects (anchors) with unique hrefs.
Hli.js-product-grid-item a.c-product-tile__pdp-link.js-product-name[href]r  r   (li.js-product-grid-item a[href$='.html']z
//a[@href])setfind_elementsr   CSS_SELECTORget_attributer<   rb   r  r  addXPATH)re  anchorsseenelementsrp   r  	elements2s   &      r-   _find_product_anchorsr    s    GUD''OOV
 A/552<<> 4<%%q! ((OOG
	 A/552<<> 4<%%q!  	))"((LAOOF399r@@BD t|#D))NN1%HHTN B N7NS         ! D  	N	s   %G6 G"G"G6 %G6  &G6 '%H H%H6H >H &H #I  ,H-H-I  I  8&I  "G3/G6 2G33G6 6HHHH HH H*)H*-H=:I  <H==I   IIc                   V ^8  d   QhRR/# r'   r*   r  r+   )r,   s   "r-   r.   r.     s     = =y =r/   c                6    V P                   ;'       g    \        p V P                  ;'       g    RpV'       g   . # \	        4       p \        VR4      pVP                  R4       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  VP                  R4       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  VP                  RRR7       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  \        P                  ! V4      p. R
OpV Ft  p	\        P                  ! W\        P                  R	7       FG  p
V
P!                  ^ 4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  Kv  	  . p\	        4       pV FK  p\#        W4      p\%        V4      '       g   K!  W9   d   K)  VP                  V4       VP'                  V4       KM  	  V#   \         d    \        p ELUi ; i  \         d    Rp ELRi ; i  \         d     EL-i ; i)zIFallback link discovery when clickable anchors are sparse or JS-rendered.r   html.parserr	  r  r
  rp   T)r  r  )zhttps?://[^\s\"'<>]+zT/(?:footwear-rwbr|other-ppe|accessories-[a-z0-9\-]+|safety-boot|product)/[^\s\"'<>]+z"/\d{3,6}\.html(?:[?#][^\s\"'<>]*)?)current_urlCATALOG_URL_PRIMARYrb   page_sourcer  r   selectra   r<   r  find_allr  r  r:   r  r  r  r   r  r  )re  base_urlr  
candidatessouprp   r  blobpatternspatr  r  outr  r  abs_urls   &               r-   '_extract_product_links_from_page_sourcer'    s*   '%%<<)<!!''R 	5JT=1ghAEE&M''R..0Dtt$ i
 GHAEE&M''R..0Dtt$ I
 s.AEE&M''R..0Dtt$ / T"DH
 Sbmm<A!!r((*Aqq! =  CUD((((?

7  Js  '&'
  4  sd   I I I5 I5 7J	  J	 A J	 J	 7AJ	 :J	 J	 I21I25JJ	JJc                    V ^8  d   QhRRRR/# )r'   
target_urlr   r*   rA   r+   )r,   s   "r-   r.   r.     s     a a a ar/   c                T   \        4        \        W4       \        V 4       \        P                  R3\        P                  R3\        P                  R3\        P                  R3\        P                  R3\        P                  R3.p\        P                  R3\        P                  R3\        P                  R	3\        P                  R
3.p\        P                  R3\        P
                  R3\        P
                  R3.p \        V 4      '       d   R#  RpRpV F1  w  rx V P                  Wx4      p	V	P                  4       '       d   T	p MK3  	  V F1  w  rx V P                  Wx4      p	V	P                  4       '       d   T	p MK3  	  V'       d	   V'       g   R#  VP                  4        VP                  \        4        VP                  4        VP                  \        4       Rp
V FY  w  rx V P                  Wx4      pVP                  4       '       d-   VP                  4       '       d   VP                  4        Rp
 MKY  K[  	  V
'       g    VP                  \         P"                  4       \$        P$                  ! 4       ^,           p\$        P$                  ! 4       V8  dp   \        V 4        \        V 4      '       d   R#   V F.  w  rxV P                  Wx4      pVP                  4       '       g   K.   M	  R# \$        P&                  ! R4       K  R#   \         d     ELi ; i  \         d     EK  i ; i  \         d     EK  i ; i  \         d     ELi ; i  \         d     ELi ; i  \         d     EK  i ; i  \         d     Li ; i  \         d     R# i ; i)zCAttempt a best-effort login for order.redwingshoes.com if prompted.zinput[type='email']zinput[name='email']zinput[name='username']zinput[id*='email' i]zinput[id*='user' i]zinput[autocomplete='username']zinput[type='password']zinput[name='password']zinput[id*='password' i]z&input[autocomplete='current-password']zbutton[type='submit']z//button[contains(translate(.,'LOGINSGNIN','loginsgnin'),'login') or contains(translate(.,'LOGINSGNIN','loginsgnin'),'sign in')]z//input[@type='submit']NFTg?)ru  r  dismiss_popupsr   r  r  r  rb   r  is_displayedclear	send_keysr   r   
is_enabledclickr   ENTERr\   rp  )re  r)  login_user_selectorslogin_pass_selectorssubmit_selectorsuser_elpass_elbyselr   	submittedbtnendpes   &&            r-   ensure_logged_inr=    s_   OV 6 
/0	/0	23	01	/0	:; 
23	23	34	BC	 
12	  V  	W	,- (( )
 GG'	&&r/D  "" # ( (	&&r/D  "" # ( ' &' &'I#	%%b.C!!cnn&6&6		 	 '7! $ $**% ))+
C
))+
v	$V,, -	/((1??$$ 0
  	

3! g    		  		  
    		  		  		s   L %)L*)L=M M" 3<M40M43N +N 8N :N L'&L'*L:9L:=MMMM"M10M14NNNNN'&N'c                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.   i  s      T r/   c                6   \         P                  R3\         P                  R3\         P                  R3\         P                  R3\         P                  R3.pV F  w  r# \	        V ^
4      P                  \        P                  ! W#34      4      pV P                  RV4       \        P                  ! R4        VP                  4        \        P                  ! R	4        R
# 	  R
#   \         d    T P                  RT4        L<i ; i  \         d     K  i ; i)zCOpen the Footwear section after login when required by the site UI.zfootwear-labelz#footwear-labelz;//button[@id='footwear-label' or @aria-controls='footwear']z//a[@id='footwear-label']zU//*[self::button or self::a or @role='tab'][contains(normalize-space(.), 'Footwear')]?arguments[0].scrollIntoView({block:'center', inline:'center'});333333?arguments[0].click();g?N)r   IDr  r  r   untilECelement_to_be_clickablerk  r\   rp  r0  rb   )re  	selectorsr7  r8  els   &    r-   open_footwear_sectionrI  i  s     
 !	+,	PQ	./	jkI 	vr*001K1KRI1VWB!!"ceghJJtC
 JJsO   C%%&=rBC  		s7    AD	9C&	D	&DD	DD		DDc                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.     s     & &4 &r/   c                   \         P                  R3\         P                  R3\         P                  R3.pRpV Fq  w  r4 \        V ^4      P                  \        P
                  ! W434      4      pV P                  RV4       \        P                  ! R4        VP                  4        Rp M	   \        V \        4        \        V ^4      P                  R	 4       R
#   \         d    T P                  RT4        LUi ; i  \         d     K  i ; i  \         d     Lai ; i  \         d)     \        T \        4        R
#   \         d      R
# i ; ii ; i)zNLegacy helper kept for compatibility; redirects to configured primary catalog.z%//a[contains(@href,'/footwear-rwbr')]z0//button[contains(@data-target,'footwear-rwbr')]zx//*[self::a or self::button][contains(normalize-space(.), 'Red Wing') and not(contains(normalize-space(.), 'Heritage'))]Fr@  rA  rB  Tc                    \        V P                  4      P                  ;'       g    R P                  4       \        \        4      P                  ;'       g    R P                  4       8H  # r   )r   r  r?   r  r  r   s   &r-   <lambda>-open_red_wing_footwear_page.<locals>.<lambda>  sG    x.3399r@@BxPcGdGiGiGoGomoFvFvFxxr/   N)r   r  r   rD  rE  rF  rk  r\   rp  r0  rb   r  r  )re  rG  clickedr7  r8  rH  s   &     r-   open_red_wing_footwear_pagerQ    sT    
:;	EF	  N  	OI
 G	vq)//0J0JB90UVB!!"ceghJJtC
 G ",-fb!''x	
  C%%&=rBC  		    	V01 		ss    ADC)D0D D% C?<D>C??DDDD"!D"%E1EEEEEc                   V ^8  d   QhRR/# r  r+   )r,   s   "r-   r.   r.     s     L L) Lr/   c                  a  V P                   ;'       g    \        p. p. ROpV Fo  p V P                  \        P
                  V4       FG  pVP                  R4      ;'       g    RP                  4       pV'       g   K6  VP                  V4       KI  	  Kq  	   V P                  \        P
                  R4       F  pR F  pVP                  V4      ;'       g    RP                  4       o\        ;QJ d#    V3R l\         4       F  '       g   K   RM	  RM! V3R l\         4       4      '       g   Kv  VP                  S4       K  	  K  	   V P                  ;'       g    RpV'       Ed   \        P                  ! V4      p	\        P                  ! RV	\        P                   R7       F#  p
VP                  V
P#                  ^ 4      4       K%  	  \        P                  ! R	V	\        P                   R7       F#  p
VP                  V
P#                  ^ 4      4       K%  	  \        P                  ! R
V	\        P                   R7       F#  p
VP                  V
P#                  ^ 4      4       K%  	  . p\%        4       pV F  p\'        W4      p \)        V4      pVP*                  ;'       g    RP-                  4       pVP.                  ;'       g    RP-                  4       p\0        T9  d   Ko  \        P2                  ! RT4      '       g    \        P2                  ! RT4      '       g   K  TP4                   RTP*                   T 2pTT9  g   K  TP7                  T4       TP                  T4       K  	  \8         F  pW9  g   K  VP                  V4       K  	  V#   \         d    \        p ELi ; i  \         d     EK  i ; i  \         d     ELi ; i  \         d    Rp ELi ; i  \         d     EK  i ; i)zEDiscover non-footwear catalog paths from RW site navigation/elements.r  r   z&[data-target], [data-url], [data-href]c              3  H   <"   T F  qSP                  4       9   x  K  	  R # 5ir   r  )r  r  r(   s   & r-   r  5discover_non_footwear_catalog_urls.<locals>.<genexpr>  s     M5LTqwwy(5L   "TFz/other-ppe(?:[/?#]|$)r  z/accessories-[a-z0-9\-]+z:https?://[^\s\"'<>]*/(?:other-ppe|accessories-[a-z0-9\-]+)z/other-ppe$z/accessories-[a-z0-9\-]+$z://)za[href*='/other-ppe']za[href*='/accessories-']znav a[href*='other-ppe']znav a[href*='accessories-'])zdata-targetzdata-urlz	data-href)r  r  rb   r  r   r  r  r<   r  r  r  r  r  r  r:   r  r  r  r  r   r   netlocr  r?   	SITE_HOSTr  schemer  r[   )re  r  r   dom_selectorsr8  rH  r  attrr  r"  r  urlsr  r  r  r   hostr?   normr(   s   &                  @r-   "discover_non_footwear_catalog_urlsr`    s,   '%%<<)< JM 	**2??C@((066B==?4%%d+ A &&r8`aB@%%d+11r88:3M5LM333M5LMMM%%a( A b!!''R t  &5t2==QAaggaj) R8$bmmTAaggaj) UZ\`hjhuhuvAaggaj) w DUDH"	AHHNN))+DFFLLb'')D D IInd++yy5t<<((3qxxj/tHHTNKK% * =KKN  KQ  '&'$  		    &  		s   N N :N2$N2?N2?O O 5O O 4O O !O 
AO+O+N/.N/2OOOOO('O(+O;:O;c                    V ^8  d   QhRRRR/# )r'   r  r`   r*   rA   r+   )r,   s   "r-   r.   r.     s      1  1  1d  1r/   c                F    R R lp\        W4      P                  V4       R# )zUWait for shared catalog-page elements instead of requiring product links immediately.c                   V ^8  d   QhRR/# ry   r+   )r,   s   "r-   r.   9wait_for_non_footwear_catalog_ready.<locals>.__annotate__  s     C CT Cr/   c                d  a   S P                  \        P                  R4      pV'       d   R#   S P	                  \        P
                  R4      P                  ;'       g    RP                  4       pRT9   pRT9   pRT9   pR	T9   pR
T9   pRT9   p\        ;QJ d#    T 3R l\         4       F  '       g   K   RM	  RM! T 3R l\         4       4      p	T;'       g5    T	;'       d+    T;'       g!    T;'       g    T;'       g    T;'       g    T#   \         d     Li ; i  \         d     R# i ; i)u  Check that the catalog UI has loaded.

On the current Red Wing site, the grid of products is present in
``li.js-product-grid-item`` elements.  Earlier heuristics looked for
text such as "Sort By", "Results" or "Filters"; those still apply
but may not appear until the user scrolls.  We consider the catalog
ready when either at least one product tile exists or legacy text
heuristics are satisfied.
zli.js-product-grid-itemTr~  r   Fresultzsort byfilterszmore resultsr  zpage not foundc              3  p   <"   T F+  qSP                   ;'       g    R P                  4       9   x  K-  	  R# 5i)r   N)r  r  )r  r  r   s   & r-   r  Fwait_for_non_footwear_catalog_ready.<locals>._ready.<locals>.<genexpr>  s+     "mUlTAMM,?,?R+F+F+H#HUls   66)
r  r   r  rb   r  r  r@   r  r  r  )
r   tilestxthas_resultshas_sorthas_filtershas_more
has_searchhas_page_not_foundhas_non_footwear_ctxs
   f         r-   _ready3wait_for_non_footwear_catalog_ready.<locals>._ready  sL   	OOBOO5NOE 	>>"++v6;;AArHHJC #o#3&!S(_
-4"s"mUl"msss"mUl"mm!  C  C&:  'B  'B  AA  AAK  AA  AA[f  AA  AAjr  AA  AA  wA  	C  		  		s(   'D 0D   D  DD D/.D/N)r   rD  )re  r  rs  s   && r-   #wait_for_non_footwear_catalog_readyru    s    C< &"((0r/   c               $    V ^8  d   QhRRRRRR/# )r'   r  r   rw  r*   zTuple[str, str, str]r+   )r,   s   "r-   r.   r.     s'     C* C*3 C*S C*=Q C*r/   c                   \        V R4      p\        V4      pRpRpVP                  RRR7      ;'       g    VP                  RR7      pV'       d   VP                  R4      pVP                  R	4      pV'       dD   VP                  R
RR7      p\        P
                  ! RV4      p	V	'       d   V	P                  ^4      pV'       d   VP                  R
RR7      pV'       EgW   VP                  R4       EF@  p
 \        P                  ! V
P                  RR7      4      p\        T\        4      '       d   TMT.pT F  p\        T\        4      '       g   K  T'       g4   TP                  R4      ;'       g    RP                  4       pT'       d   TpT'       gn   R Fg  p\!        TP                  T4      ;'       g    R4      P                  4       p\        P
                  ! RT4      pT'       g   KV  TP                  ^4      p M	  T'       g   K  T'       g   K   M	  T'       g   EK6  T'       g   EKA   M	  V'       g   R F  p VP#                  V4      pV'       g   K  VP%                  R4      '       d+   VP                  R4      ;'       g    RP                  4       pMVP                  R
RR7      pV'       g   K}  RVP'                  4       9  g   K  Tp M	  V'       g   V'       d   RV 2MT;'       g    RpV'       g   VP                  4       pV'       g   TpW4V3#   \         d     EK?  i ; i  \         d    Rp Li ; i)zMExtract (style_number, name, style_text) from legacy and RW order-site pages.r  r   r  	shoeguide)class_printSpacing)idr  strongr8   Tr<   z	#\s*(\d+)z"script[type='application/ld+json']r   \b(\d{3,6})\bNmetacontentzred wing for businessStyle #)r  mpn	productID)r  r  r   zmeta[property='og:title'])r   extract_style_from_urlfindget_textr:   r  r  r  r]   r^   rb   
isinstancelistdictra   r<   r   
select_oner  r  )r  rw  r!  style_numberr   
style_text	guide_divh3_tag
strong_tagmatchscriptpayloadobjsobjnmr  r  r  r8  rH  rk  s   &&                   r-    extract_style_and_name_from_htmlr    s   }-D)#.LDJ		%	4TT		^	8TI%^^H-
D9JIIlJ7E${{1~&&s$&7D4kk"FGF**V__4_%@A )$777gYD!#t,,''&///R668B!#:!#''#,"4"4"5;;=II&6<1+,771:L!  ; 4LL   t/ H2 EC__S) ~~f%%vvi(..B557kk#Tk2s.ciikA F 1=w|n-DJJB
!'')z))Y  4  s$   8&L#L6#L32L36MMc                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.   `  s      d r/   c                   . ROp\         P                   ! 4       R,           p\         P                   ! 4       V8  d   RpV Ff  p V P                  \        P                  V4      pVP	                  4       '       d-   VP                  4       '       d   VP                  4        Rp MKf  Kh  	  V'       g   R# \         P                  ! R4       K  R#   \         d     K  i ; i)N//button[contains(.,'Agree') or contains(.,'Accept') or contains(.,'Proceed')]g       @FT皙?N)r  zI//a[contains(.,'Agree') or contains(.,'Accept') or contains(.,'Proceed')]uR   //button[contains(.,'Close') or contains(.,'×') or contains(@aria-label,'Close')])	r\   r  r   r  r,  r/  r0  rb   rp  )re  xpathsr;  rP  xprH  s   &     r-   r+  r+  `  s    F
 ))+
C
))+
B((26??$$HHJ"G *9$  

3   s   ACCCCc                    V ^8  d   QhRRRR/# )r'   
max_roundsr`   r*   rA   r+   )r,   s   "r-   r.   r.   w  s     B B3 B Br/   c           	        ^ pRp\        V4       EF  p\        V 4       RpR F  p V P                  \        P                  V4      pVP                  4       '       g   K<  VP                  R4      p\        VR4      '       d   VP                  4       '       g   Kw  VR9  d   K   V P                  RV4       \        P                  ! R4        VP                  4        Rp\        P                  ! R	4        M	  \        V 4      p	\        V	 U
u0 uF,  qP                  R
4      '       g   K  V
P                  R
4      kK.  	  up
4      pW8X  d   V^,          pM^ pTpV'       g   V^8  d    R#  V P                  R4       \        P                  ! R4       EK  	  R#   \         d     Li ; i  \         d+     T P                  RT4        L  \         d      EK  i ; ii ; i  \         d     EK  i ; iu up
i   \         d     Li ; i)r{   Fdisabledr/  Nr@  rA  rB  T333333?r  z/window.scrollTo(0, document.body.scrollHeight);g      ?)zz//button[contains(translate(normalize-space(.),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'more results')]zu//a[contains(translate(normalize-space(.),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'more results')])Nfalser   )r  r+  r  r   r  r,  r  r  r/  rk  rb   r\   rp  r0  r  r  )re  r  stable_rounds
last_count_clicked_morer  r:  disabled_attrlinksrp   counts   &&          r-   scroll_to_load_allr  w  s   MJ:v
B!))"((B7'')) # 1 1* =3--cnn6F6F (;;))Y 

4 !IIK  $

3G
N &f-eWev?V,Q__V,eWXQMMJ  2	!!"ST 	

3 8 ! 
 ! !!--.EsK$ ! !	!   X  		s   6G$8G$G$$F6G$F,G$G7
*G7
-G<F)&G$(F))G$,G!8G
G$G	G!G$G	G!!G$$G43G4<H
	H
c                    V ^8  d   QhRRRR/# r3   r+   )r,   s   "r-   r.   r.     s      3 3 r/   c                $   T ;'       g    R P                  4       p \        P                  ! RRV 4      p V '       g   R # V P                  4       pVR9   d   R # \        P                  ! RV 4      '       d   R # RV 9   d   R # \        V 4      ^8  d   R # V # )r   r7   r8   z\d{2,}$>   r  r  r  r  
QUICK VIEWADD TO CARTVIEW DETAILSr!   )r<   r:   r;   r  r  r  )r4   r  s   & r-   _clean_listing_namer    sz    	
bA
vsAA		A 	 	 	||Iq!!
ax
1vzHr/   c                   V ^8  d   QhRR/# r'   r*   r   r+   )r,   s   "r-   r.   r.     s     1 1C 1r/   c                   . p TP                  V P                  R4      ;'       g    R4        TP                  V P                  R4      ;'       g    R4        TP                  V P                  ;'       g    R4       . pV FS  pV'       g   K  \	        V4      P                  4        F)  p\        V4      pV'       g   K  VP                  V4       K+  	  KU  	  \        4       p. pV F-  pWE9  g   K  VP                  V4       VP                  V4       K/  	  V'       g   R# R R lpVP                  VRR7       V^ ,          #   \         d     EL%i ; i  \         d     ELi ; i  \         d     Li ; i)zJBest-effort: read the product name as displayed on a catalog listing tile.z
aria-labelr   r   c                    V ^8  d   QhRRRR/# )r'   r4   r   r*   r`   r+   )r,   s   "r-   r.   /_listing_name_from_anchor.<locals>.__annotate__  s     
 
 
 
r/   c                  a  \        S 4      p\        P                  ! R S 4      '       d
   V^,          p\        P                  ! RS P                  4       4      '       d
   V^
,          p\        P                  ! RS P                  4       4      '       d
   V^,          p\        ;QJ d    V 3R lR 4       F  '       g   K   RM	  RM! V 3R lR 4       4      '       d
   V^,          pV# )z[A-Za-z]\bWOMEN'?S\b
\bMEN'?S\bc              3  H   <"   T F  qSP                  4       9   x  K  	  R # 5ir   rU  r  rC  r4   s   & r-   r  ;_listing_name_from_anchor.<locals>.score.<locals>.<genexpr>  s     X'W!AGGI~'WrW  TF)viewquickcartcomparewishlist)r  r:   r  r  r  )r4   scs   f r-   score(_listing_name_from_anchor.<locals>.score  s    V99[!$$"HB99_aggi00"HB99]AGGI..!GB3X'WX333X'WXXX"HB	r/   T)r  reverse)
r  r  rb   r@   r   
splitlinesr  r  r  sort)rp   r   linesclnr  uniqr  s   &       r-   _listing_name_from_anchorr    sK   J!//,7==2>!//'288b9!&&,,B' Ea&##%B$R(BrR  &  5DD>HHRLKKO 
 
 	II%I&7NY      sE   "E E "E E E) 8E) EEE&%E&)E76E7c               $    V ^8  d   QhRRRRRR/# )r'   rQ   List[List[str]]	preferredDict[str, str]r*   r`   r+   )r,   s   "r-   r.   r.     s!       N s r/   c                j   V '       d	   V'       g   ^ # ^ pV  F  p\        V\        4      '       d   \        V4      ^8  d   K+  V^ ,          ;'       g    RP                  4       pV'       g   KV  \	        VP                  VR4      4      pV'       g   K{  V^,          V8w  g   K  WS^&   V^,          pK  	  V# )zQOverwrite Name column using preferred mapping (by style). Returns number updated.r   )r  r  r  r<   r  ra   )rQ   r  updatedr"  r  prefs   &&    r-   apply_preferred_namesr    s    yG!T""c!fqj1""$"9==#;<4AaDDLaDqLG  Nr/   c                   V ^8  d   QhRR/# )r'   r*   z Tuple[List[str], Dict[str, str]]r+   )r,   s   "r-   r.   r.     s     <& <&%E <&r/   c           	     Z   . p\        4       p\        4       p/ p\        V \        4       \        V 4      ;'       g    \	        \
        4      p\        P                  P                  R\        V4       R24       V F&  p\        P                  P                  RV R24       K(  	  V EF  p\        4        \        W4       \        W4       \        V ^#R7       \        V 4       \        V 4       . p\        V 4       FB  p	V	P!                  R4      p
V
'       g   K  \#        V
4      '       g   K1  VP%                  V
4       KD  	  \'        V 4       F  p
W9  g   K  VP%                  V
4       K  	  \        P                  P                  RV R\        V4       R24       V F  p
W9   d   K  \)        V
4      p\*        '       d   V'       d	   R	V9   d    V'       d+   W9   d   VP-                  V
4       KS  VP-                  V4       VP-                  V
4       VP%                  V
4       K  	  EK  	  W3# )
zCollect unique product links from all configured catalog sections.

Dedupes by style number when possible (preferred), otherwise by URL.
z,[collect] non-footwear catalogs discovered: rq   z[collect] catalog: r  r  z
[collect] z -> discovered z candidate product links
zcatalog=international)r  r=  r  r`  r  r[   r   stderrrt   r  ru  r  ru  r+  r  r  r  r  r  r'  r  "PREFER_INTERNATIONAL_LISTING_NAMESr  )re  	all_links	seen_href
seen_stylerR   rN   r  rw  
page_linksrp   r  r  s   &           r-   collect_product_linksr    s   
 I%I5J&(OV015f=SSlASLJJCCDUCVVXYZ

.qc45  %+FB?v6" !#
&v.A??6*D#D))d# / <FCD%!!$' D 	

:cU/#j/9JJdefD *40E11e@W[^@^&MM$'u%MM$T"! 3 V %%r/   c                    V ^8  d   QhRRRR/# )r'   linkr   r*   r+   )r,   s   "r-   r.   r.   [  s     $ $ $ $r/   c                <   V '       g   R # \         P                  ! RV 4      pV'       d   VP                  ^4      # \         P                  ! RV 4      pV'       d   VP                  ^4      # \         P                  ! RV \         P                  R7      pV'       d   VP                  ^4      # \         P                  ! RV \         P                  R7      pV'       d   VP                  ^4      # \         P                  ! RV 4      pV'       dQ   RV P	                  4       9   g+   RV P	                  4       9   g   R	V P	                  4       9   d   VP                  ^4      #  \        \        V 4      P                  4      pR F  pVP                  V4      ;'       g*    VP                  VP	                  4       4      ;'       g    . pV F@  p\         P                  ! R
\        V4      4      pV'       g   K-  VP                  ^4      u u # 	  K  	  R #   \         d     R # i ; i)r   z/safety-boot/(\d+)[-/]z/safety-boot/(\d+)z2/footwear-[^/]+/(?:[^/?#]*/)?(\d{3,6})(?:[-/?#]|$)r  z/(\d{3,6})\.html(?:[?#]|$)z/(\d{3,6})(?:[-/?#]|$)zfootwear-rwbrz	other-ppezaccessories-r~  r  )r:   r  r  r  r  r
   r   r   ra   r   rb   )r  r  qr  valsr(   s   &     r-   r  r  [  s   
		+T2Awwqz
		'.Awwqz 			GUWUbUbcAwwqz
		/R]]KAwwqz
		+T2A4::<'$**,&TZZ\)wwqz	Xd^))*NC55:99syy{!399rDII.A71771:%  O   s*   :H &H =-H /H H HHc                    V ^8  d   QhRRRR/# r  r+   )r,   s   "r-   r.   r.     s        r/   c                  a  S ;'       g    R P                  4       P                  4       o S '       g   R# Rp\        ;QJ d    V 3R lV 4       F  '       g   K   RM	  RM! V 3R lV 4       4      '       d   R# \        S 4      ^<8  d   R# R# )r   Tc              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r+   r  s   & r-   r  #_looks_like_junk.<locals>.<genexpr>  s     
';a6;r  F)zwindow.openr  zfacebook.comzhttp://zhttps://)r<   r  r  r  )r4   junk_tokenss   f r-   _looks_like_junkr    s^    	
b!AWK
s
';
'sss
';
'''
1v{r/   c                    V ^8  d   QhRRRR/# )r'   rowr  r*   r)   r+   )r,   s   "r-   r.   r.     s      Y 4 r/   c                     V ^ ,          ;'       g    RP                  4       pV ^,          ;'       g    RP                  4       pT'       g   R# TP                  4       \        8X  d   R# R#   \         d     R# i ; i)r{   r   TF)r<   rb   r  BAD_NAME_SENTINEL)r  r  r   s   &  r-   _is_bad_rowr    sh    Q2$$&A"##% zz|((  s   A+  A+ A+ +A:9A:c                    V ^8  d   QhRRRR/# r'   rG  r   r*   r`   r+   )r,   s   "r-   r.   r.     s     & &T &c &r/   c                   V P                  R4      '       d   ^ # V P                  R. 4      ;'       g    . p\        V P                  R. 4      ;'       g    . 4      p^ p\        4       pV EF"  p\        V\        4      '       d	   V'       g   K$  V^ ,          ;'       g    RP	                  4       pV'       g   KO  \        V4      \        8  d   VP                  V4       Kv  \        V4      ^8  d"   V^,          ;'       g    RP	                  4       MRp\        V4      ^8  d"   V^,          ;'       g    RP	                  4       MRpV'       g   VP                  V4       K  \        '       g   EK  V'       d   EK  VP                  V4       EK%  	  V'       g   ^ # \        V4       F:  p	\        V	4      p
V
'       g   K  W9   g   K   VP                  V	4       V^,          pK<  	  \        V4      V R&   V# )zIf checkpoint rows are missing URL/Image columns or have empty URL/Image, requeue those links.
Runs at most once per checkpoint unless you delete/clear ck['media_repair_done'].
rS   rQ   rP   r   )ra   r  r  r  r<   r  EXPECTED_COLSr  CAPTURE_IMAGESr  discardsorted)rG  rQ   rP   removedstyles_neededr"  r  url_cellimg_cellr  rD  s   &          r-   repair_missing_mediar    sz    
vv!""66&"##DRVVL"-334JGEM!T""!1""$q6M!e$+.q6A:AaDJJB%%'2+.q6A:AaDJJB%%'2e$>((e$ " Z #D)2"%t$qLG	 ! j)B|Nr/   c                    V ^8  d   QhRRRR/# r  r+   )r,   s   "r-   r.   r.     s     - -4 -C -r/   c                    \        V P                  R. 4      ;'       g    . 4      p\        V P                  R. 4      ;'       g    . 4      pV'       d	   V'       g   ^ # V Uu0 uF(  q3'       g   K  \        V4      '       g   K  V^ ,          kK*  	  ppV'       g   ^ # ^ p\        V4       F:  p\	        V4      pV'       g   K  Wt9   g   K   VP                  V4       V^,          pK<  	  V'       d   \        V4      V R&   \        V P                  R/ 4      ;'       g    / 4      p\        VP                  4       4       F2  p	\	        V	4      pV'       g   K  Wt9   g   K   VP                  V	R4       K4  	  WR&   V P                  R. 4      ;'       g    .  U
u. uF  p
\	        V
4      V9  g   K  V
NK  	  up
V R&   \        V 4       V# u upi u up
i   \         d     ^ # i ; i)aM  If checkpoint contains obviously bad rows, un-mark those links as 'done'.

This fixes the situation where a previous run captured the site header/ads into the Name/Brand
columns and those rows are now 'stuck' because resume logic skips already-done links.

Returns:
    Number of links that were re-queued (removed from done_links).
rQ   rP   rT   NrU   )r  ra   r  r  r  remover  r  keyspoprl   rb   )rG  rQ   rP   r"  
bad_stylesr  r  rD  fckr  s   &          r-   repair_bad_checkpoint_rowsr    s   $BFF62&,,"-b177R8
:$(ADqAd+a.daddD
A$D'-Brb&!!$'1	 % %j1B| bff]B/5526B"'')_+A.2"*FF1dO % !#} 4666:Mr3R3X3XVX3X 'V3XQ)?)B*)T ()q3X 'VB"# B9 B.'V  s   G $G G G G "	G0GG
G  G G 	%G /+G 3G G 2G G G*G0G 
G GGc               $    V ^8  d   QhRRRRRR/# )r'   
text_blockr   
field_namer*   r+   )r,   s   "r-   r.   r.     s!       # # r/   c           
        T ;'       g    RP                  4        Uu. uF  q"P                  4       NK  	  pp\        V4       F  w  rEV'       g   K  VP                  V4      '       g   K(  V\	        V4      R P                  R4      pV'       d   Vu # \        V^,           \        V^,           \	        V4      4      4       F1  pW7,          ;'       g    RP                  4       pV'       g   K-  Vu u # 	   R# 	  R# u upi )u   Parse a simple field/value from extracted page text.

Red Wing pages often render fields in tables so the extracted text looks like:

    Name
    DynaForce®

We support both 'Name: DynaForce' and 'Name' on one line with the value on the next.
r   Nz :	)r  r<   	enumerater  r  r  r   )	r  r  r  r  irn   tailjnxts	   &&       r-   parse_field_liner    s     $.#3#3"?"?"AB"ABXXZ"AEBU#??:&&J()//7D1q5#a!eSZ"89x~~2,,.3J :  $  Cs   C<c               $    V ^8  d   QhRRRRRR/# )r'   header_textr   	body_textr*   r+   )r,   s   "r-   r.   r.     s!      s s s r/   c                   T ;'       g    R P                  4       p\        P                  ! RV\        P                  R7      pV'       d   VP	                  ^4      P                  4       pV'       db   . pVP                  4        F:  pTP                  VP                  4       '       d   TMVP                  4       4       K<  	  RP                  V4      # VP                  4        F  p\        P                  ! RV\        P                  R7      '       g   K2  \        P                  ! RVP                  4       \        P                  R7      pV'       d   VP	                  ^4      P                  4       pV'       dd   . pVP                  4        F:  pTP                  VP                  4       '       d   TMVP                  4       4       K<  	  RP                  V4      u #  R # 	  R # )r   z^(.*?)\s+style\s*#\s*\d+r  r8   z\bstyle\s*#\s*\d+\bz^(.*?)\s+style\s*#\s*\d+\b)r<   r:   r  r  r  splitr  isupper
capitalizer  r  )	r  r  htr  r  r%  r   rn   m2s	   &&       r-   extract_brandr    s<   


	"	"	$B
		-rGAggaj CYY[

		1@ !88C= $$&99+TGG8$**,bmm\Bhhqk'')C YY[

		1H )88C=( ' r/   c               $    V ^8  d   QhRRRRRR/# )r'   r  r   r  r*   r+   )r,   s   "r-   r.   r.   1  s!     	& 	&# 	&c 	&c 	&r/   c                   V'       dY   V P                  4       P                  R V 24      pVR8w  d0   WVR,            p\        VR4      pV'       d   VP                  4       # \        V R4      pV'       d   VP                  4       # R# )z
ABOUT THE i@  Namer   r  )r  r  r  r<   )r  r  idxchunkvals   &&   r-   extract_about_namer  1  sr    oo$$z%%9:"93:.E"5&1Cyy{"
9f
-C399;%2%r/   c                    V ^8  d   QhRRRR/# )r'   r@   r   r*   zDict[str, bool]r+   )r,   s   "r-   r.   r.   =  s     9 9 9 9r/   c           	     0   \        R  \        P                  ! RV P                  4       4       4       4      pV\        R \        P                  ! RV P                  4       4       4       4      ,          p\	        ^^4       Uu/ uF
  q" R2W!9   bK  	  up# u upi )c              3  8   "   T F  p\        V4      x  K  	  R # 5ir   r`   r  r   s   & r-   r   parse_heights.<locals>.<genexpr>>  s     V U1A U   z\b(\d{1,2})\s*-\s*INCH\bc              3  8   "   T F  p\        V4      x  K  	  R # 5ir   r"  r#  s   & r-   r  r$  ?  s     S!RAQ!Rr%  z\b(\d{1,2})\s+INCH\b")r  r:   findallr  r  )r@   foundr	  s   &  r-   parse_heightsr*  =  st    V

+F

 UVVE	SS,CTZZ\!RSSSE+0B<8<acGaj <888s    Bc                    V ^8  d   QhRRRR/# )r'   	brand_strr   r*   zTuple[bool, bool, bool]r+   )r,   s   "r-   r.   r.   C  s     ! !S !-D !r/   c                    T ;'       g    R P                  4       pVP                  R4      pVP                  R4      pVP                  R4      pRV9   d!   VP                  R4      '       g
   RV9   d   RpW#V3# )r   zred wingzirish setterworxzby red wingT)r  r  )r,  bis_rwis_isis_worxs   &    r-   classify_brand_familyr3  C  sl    	b!ALL$ELL(Ell6"G!all622!9K  r/   c               $    V ^8  d   QhRRRRRR/# )r'   r   r   max_lenr`   r*   r+   )r,   s   "r-   r.   r.   R  s!      3  s r/   c                   T ;'       g    R P                  4       p\        P                  ! RRV4      pVP                  RR 4      p\        P                  ! RR V\        P                  R7      pVP                  4       P                  RR4      p\        P                  ! RRV4      pV'       g   Rp\        V4      V8  d   VR	V P                  R4      pV# )
r   r7   r8      ®z[^\w\-\.\s]+r  r  z_+bootN)r<   r:   r;   r9   UNICODEr  ru   )r   r5  r4   s   && r-   _safe_filename_from_namer:  R  s    	A
vsAA			$A
ARZZ8A		#s#A
uc1A
1vhwKs#Hr/   c                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.   `  s      T r/   c                     \         P                  ! 4       P                  R ,          P                  4       p V P	                  RRR7       V # )ImagesTr   )r   cwdparentresolver   rM  s    r-   _images_dirrA  `  s6    			X	%..0AGGD4G(Hr/   c                    V ^8  d   QhRRRR/# )r'   fnamer   r*   r+   )r,   s   "r-   r.   r.   f  s     . .3 .3 .r/   c                F    \        \        R 4      R,          V ,          4      # )z..r=  )r   r   )rC  s   &r-   _rel_image_pathrE  f  s    tDzH$u,--r/   c                  aaa R FH  p V P                  \        P                  V4      pV'       d   VP                  4       '       d   Vu # KH  KJ  	   V P                  \        P
                  R4      pVP                  \        P                  R4      pV'       d   VP                  4       '       d   V#  T P                  \        P                  R4      pT'       g"    T P                  \        P                  R4      pRpRpT EF  p TP                  4       '       g   K  TP                  ;'       g    / p\        TP                  R4      ;'       g    ^ 4      p	\        TP                  R4      ;'       g    ^ 4      p
T	^x8  g   T
^x8  d   K  TP                  R4      ;'       g    R	P                  4       oTP                  R
4      ;'       g    R	P                  4       oTP                  R4      ;'       g    R	P                  4       oY,          p\        ;QJ d    T3R lR 4       F  '       g   K   RM	  RM! T3R lR 4       4      '       d
   TR,          p\        ;QJ d    T3R lR 4       F  '       g   K   RM	  RM! T3R lR 4       4      '       d
   TR,          p\        ;QJ d    T3R lR 4       F  '       g   K   RM	  RM! T3R lR 4       4      '       d
   TR,          pY8  d   TpTpEK  EK  	  T#   \         d     EK	  i ; i  \         d     ELni ; i  \         d    . p ELai ; i  \         d    . p ELLi ; i  \         d     EKY  i ; i)@div.slick-slide.slick-current.slick-active img[itemprop='image']productImageimgmain imgNr   r   heightaltr   classsrcc              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r+   )r  r  rL  s   & r-   r  1_pick_best_product_img_element.<locals>.<genexpr>  s     P%O8%Or  TFr  c              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r+   )r  r  clss   & r-   r  rP    s     M%L8%Lr  gffffff?c              3  ,   <"   T F	  qS9   x  K  	  R # 5ir   r+   )r  r  rN  s   & r-   r  rP    s     R%Q8%Qr  g?)rG  .div.slick-slide.slick-current.slick-active imgHdiv.c-image-carousel__slider-item.js-carousel-item img[itemprop='image'])r8  shoechukkahikermoc)productprimaryheroimage)z	/dw/imagescene7z/imagesstatic)r  r   r  r,  rb   rC  r  r  sizer   ra   r  r  r  )re  r8  rI  	containerimgsbest
best_scorerH  szr   hr  rL  rR  rN  s   &           @@@r-   _pick_best_product_img_elementrg  j  s   
	%%boos;Cs''))
 *s''~>	$$R__e<3##%%J
##BOOZ@ 	''U;D DJ	??$$BBbffWo**+AbffX&++!,A3w!c'##E*00b779C##G,2299;C##E*00b779CEEsP%OPsssP%OPPPsM%LMsssM%LMMMsR%QRsssR%QRRR!"
 "' 0 Kc  		    
  	D	6  		s   =L L AL L 1L 4 L%  L9 M M3M#M7MM&(M(M88M3MM%M<MM.MM#M LLL"!L"%L65L69M
	M
MMc                   V ^8  d   QhRR/# r  r+   )r,   s   "r-   r.   r.     s     1 1# 1r/   c                r   R F  p V P                  \        P                  V4      pVP                  R4      ;'       g    RP	                  4       pVP                  R4      ;'       g    RP	                  4       pV'       g	   V'       d   T;'       g    Tu # K  	   V P                  \        P                  R4      p VP                  \        P                  R4      pV'       do   VP                  R4      ;'       g    RP	                  4       pVP                  R4      ;'       g    RP	                  4       pV'       g	   V'       d   T;'       g    T# R F  p T P                  \        P                  T4      pTP                  R4      ;'       g    RP	                  4       pTP                  R4      ;'       g    RP	                  4       pT'       g	   T'       d   T;'       g    Tu # K  	  R#   \
         d     EK  i ; i  \
         d    Rp EL5i ; i  \
         d     Li ; i  \
         d     K  i ; i)	rG  zoomimgr   rN  rH  rI  N)rG  rT  rU  z6div.c-image-carousel__slider-item.js-carousel-item img)z$li[data-orbit-slide="product-1"] imgzli.active imgzul#productImage li.active imgzmain li.active imgrJ  )r  r   r  r  r<   rb   rC  )re  r8  rI  rA  r4   ra  s   &     r-   _extract_product_image_urlrk    s   	%%boos;C""9-33::<A""5)//R668AAvvA  ''~>		((%@C ""9-33::<A""5)//R668AAvvA
	%%boos;C""9-33::<A""5)//R668AAvvA   G  		  	C	  "  		s   7G/(G/(G/G/	G/G/ H = H H %H =(H &H ?H H H 7H'(H':H'H'H'$H'/G?>G?HH HH H$#H$'H65H6c                    V ^8  d   QhRRRR/# )r'   r  r   r*   r+   )r,   s   "r-   r.   r.     s     8 8c 8c 8r/   c                   T;'       g    RP                  4       pV'       g   R# RV R2p\        4       V,          p \        V 4      pV'       dy   VP                  RR4      pRR/p\        P
                  ! WE^R7      pVP                  '       d9   VP                  '       d'   VP                  VP                  4       \        V4      # R	p \        T 4      pT'       g   R#  T P                  R
T4       \        P                  ! R4        \        T 4        TP!                  \#        T4      4       \        T4      #   \         d     Li ; i  \         d    R	p Li ; i  \         d     Lci ; i  \         d     Lhi ; i  \         d=     T P%                  \#        T4      4       \        T4      u #   \         d      R# i ; ii ; i)zSave the main product image as a temp file (__<style>.png).

Preferred: download the product image URL (zoomimg/src) for best quality.
Fallback: element screenshot if download fails.
Returns relative path like ../Images/__595.png (or empty string).
r   __.pngz&amp;&r  zMozilla/5.0)r  r  Nr@  r  )r<   rA  rk  r9   requestsra   okr  write_bytesrE  rb   rg  rk  r\   rp  r+  
screenshotr   save_screenshot)re  r  rC  out_pathimg_urlr  r"  img_els   &&      r-   capture_product_image_tempry    s    [[b!EtE}u$H
,V4oogs3G#]3GWrBAttt			$$QYY/&u--
 F/7 _agh

3v#h-(u%%5      
    	""3x=1"5)) 			s~   A$D; "%D; E !(E 
E0 $F ;E	E	EEE-,E-0E>=E>G$F41G4G?GGGc               (    V ^8  d   QhRRRRRRRR/# )r'   r  r   	boot_namerel_temp_pathr*   r+   )r,   s   "r-   r.   r.     s(     " "3 "3 "s "s "r/   c                   V'       g   R# \         P                  ! 4       V,          P                  4       pVP                  4       '       g   V# \        P
                  ! RRT ;'       g    RP                  4       4      pV'       g   V# V R2p\        4       V,          p VP                  V4       \        T4      #   \         d    Tu # i ; i)z8Rename ../Images/__<style>.png -> ../Images/<style>.png.r   z[^\dA-Za-z_-]ro  )r   r>  r@  rY   r:   r;   r<   rA  r9   rb   rE  )r  r{  r|  temp_absstyle_cleanrC  dest_abss   &&&    r-   finalize_image_filenamer    s    
]*335H??&&)2/B/B/DEKm4 E}u$H" 5!!  s   B: :C
	C
c               $    V ^8  d   QhRRRRRR/# )r'   r  r   
source_urlr*   zTuple[bool, bool]r+   )r,   s   "r-   r.   r.   .  s"       # 2C r/   c                   T ;'       g    RP                  4       pT;'       g    RP                  4       p\        P                  ! RV4      '       g   RV9   g   RV9   g   RV9   d   R# \        P                  ! RV4      '       g   RV9   g   RV9   g   R	V9   d   R# \	        \        P                  ! R
V4      4      p\	        \        P                  ! RV4      4      pWE3# )zInfer gender flags from URL + header (do NOT use body text; it frequently contains both words).

Priority:
  1) URL slug/query (mens/womens)
  2) Header text (MEN'S / WOMEN'S)
  3) Otherwise: unknown -> (False, False)
r   z/womens(?:[-/]|$)zwomens-zgender=womenzgender=femalez/mens(?:[-/]|$)zmens-z
gender=menzgender=maler  r  )FT)TF)r  r  r:   r  r)   )r  r  huulmalefemales   &&    r-   infer_genderr  .  s     ,,B			B


	!	!	#B 
yy%r**i2oSUAUYhlnYn	yy#R((GrM\R=OS`dfSf		-,-D"))OR01F>r/   c                    V ^8  d   QhRRRR/# )r'   r   r   r*   r+   )r,   s   "r-   r.   r.   D  s     9 93 93 9r/   c                t    \         P                  ! R R\        T ;'       g    R4      4      P                  4       # )r7   r8   r   )r:   r;   r   r<   )r   s   &r-   _clean_pdp_textr  D  s)    66&#s5;;B/06688r/   c                    V ^8  d   QhRRRR/# r'   r!  r   r*   r   r+   )r,   s   "r-   r.   r.   H  s      - C r/   c                    R FF  pV P                  V4      pV'       g   K  \        VP                  RRR7      4      pV'       g   KD  Vu # 	  R# )%.st-product-detail__short-descriptionr8   Tr}  r   )r  z.st-product-detail__subtitle)r  r  r  )r!  selectornoder@   s   &   r-   '_extract_non_footwear_short_descriptionr  H  sG    ]x(t}}S}=>4K ^ r/   c                    V ^8  d   QhRRRR/# r  r+   )r,   s   "r-   r.   r.   S  s      M c r/   c                   R F  pV P                  V4      pV'       g   K  \        \        V4      R4      pVP                  R4       F  pVP	                  4        K  	  \        VP                  RRR7      4      pV'       g   K  Vu # 	  R# )*.st-product-detail__long-description__fullr  buttonr8   Tr}  r   )r  z-.st-product-detail__long-description__partialz$.st-product-detail__long-description)r  r   r   r  	decomposer  r  )r!  r  r  fragmentr  r@   s   &     r-   !_extract_non_footwear_descriptionr  S  s|    
 x( TM:ooh/F 0x00D0AB4K r/   c               $    V ^8  d   QhRRRRRR/# )r'   r!  r   r  r   r*   r  r+   )r,   s   "r-   r.   r.   e  s"     ( ( (# (We (r/   c           	     R   / pV P                  R 4       F  pVP                  R4      pVP                  R4      p\        WE4       F  w  rg\        VP                  RRR7      4      p\        VP                  RRR7      4      p	V'       d	   V	'       g   KO  VP	                  4       P                  R4      '       g   VP	                  4       R8X  d   K  W9  g   K  WV&   K  	  K  	  V'       g	   V'       g   V# \        T;'       g    R4      P                  4        U
u. uF  p
\        V
4      NK  	  pp
V U
u. uF  q'       g   K  V
NK  	  pp
 VP                  R4      ^,           p0 R	mpTpT^,           \        T4      8  d   Y,          pY9   d    T# Y^,           ,          p	Y9   d    T# TP	                  4       P                  R4      '       g   TP	                  4       R8X  d   T^,          pK  TP                  Y4       T^,          pK  T# u up
i u up
i   \         d    Tu # i ; i)
dldtddr8   Tr}  r  r   zPRODUCT DETAILS>   CONTACTr  SUPPORTEMAIL US)r  r  r  r  r  r'  r   r  index
ValueErrorr  rc   )r!  r  detailsr  r  valueskey_node
value_noder  r   rn   r  startstop_tokensr  s   &&             r-   %_extract_non_footwear_product_detailsr  e  s    Gkk$yy4$'$5 H!("3"3Ct"3"DEC#J$7$74$7$HIEeyy{##G,,		w0F!$ %6   i/29??/C/N/N/PQ/Pt_T"/PEQ#,edtTTeE,-.2 @KE
!)c%j
 l N ai  N 99;((CIIK7,BQJE3&
N+ R,  s$   H4	HHH H&%H&r  image_rel_pathr  c               8    V ^8  d   QhRRRRRRRRRRRRRRR	R
/# )r'   r  r   r   r  r  r  r  r  r*   r  r+   )r,   s   "r-   r.   r.     sX     $ $$
$ $ 	$ $ $ $ $r/   c          	     |   RpRpRp	V'       dQ    \        VR4      p
\        V
4      p\        V
4      p\        WR7      pV'       d   \        P
                  ! VRRR7      p	\        V 4      \        V4      \        V4      \        V4      \        V4      \        V4      \        V	4      .#   \         d
    RpRpRp	 L[i ; i)z/Build the output row for non-footwear scraping.r   r  )r  F)ensure_ascii
separators),:)r   r  r  r  r]   rk   rb   r=   )r  r   r  r  r  r  r  short_descriptiondescriptionproduct_details_jsonr!  r  s   &&&&$$$     r-   _build_row_from_textr    s     K
	& }5D G M;DAK;DVG'+zz'Zd'e$ 	utz"~&(){#+,   	& "K#% 	&s   AB' 'B;:B;c               $    V ^8  d   QhRRRRRR/# )r'   r  r   image_temp_relr*   r  r+   )r,   s   "r-   r.   r.     s!     / /3 / /Y /r/   c                j   \         \        V 4      P                  ;'       g    R 9   d   \        R4      hRp\	        V 4       F@  p \        V^R7      p\        WC4      w  rVpTp\        V4      p	\        VVVV	V VVR7      p
V
u # 	  V'       d   Vh\        R4      h  \         d   pTp Rp?Kl  Rp?ii ; i)r   z3HTTP fallback disabled for authenticated order siteNr  r  r  r  zHTTP fallback failed)
rY  r   rX  r(  r  r  r  r  r  rb   )r  r  r  r  r  r  r   r  r  r  r  rL  s   &&          r-   _scrape_product_via_httpr    s    Xd^**00b1PQQ(,H4 	q"-D&Ft&O#EF%d+I&-C J !& 
-
..  	H	s   ;BB2%B--B2prefer_httpc               $    V ^8  d   QhRRRRRR/# )r'   r  r   r  r)   r*   r  r+   )r,   s   "r-   r.   r.     s&     1M 1M 1Md 1My 1Mr/   c          
        R p\        V4      pTp\        4        V'       d    \        WR7      #  \        \        V4      P                  ;'       g    R 9   d   \        V \        4       \        W4       \        V 4       V P                  ;'       g    R p\        Wa4      w  rGp\        '       d    V'       d   \        W4      p\        V ^4      P!                  \"        P$                  ! \&        P(                  R34      4       V P+                  \&        P(                  R4      P,                  ;'       g    R p	Tp
\/        VVV
V	VVVR7      pV#   \         d     EL*i ; i  \         d    R p Li ; i  \         d    \        YR7      u # i ; i)r   )r  r~  r  )r  ru  r  rb   rY  r   rX  r=  r  r  r+  r  r  r  ry  r   rD  rE  presence_of_element_locatedr   r  r  r@   r  )re  r  r  r  r  style_from_urlr  r   r  r  r  r  s   &&$         r-   scrape_productr    sh   N"4(ENO	+DPP
#M$..44"5V%89v !!''R"B4"NZ >$%?%NN 	fb!''(F(FU[G\(]^''V<AAGGR	")
 
G  		&  $!#$$  M'LLMsZ   E  E0 AE0 /E 7E A/E0 2E0 EEE-*E0 ,E--E0 0F
	F
c                    V ^8  d   QhRRRR/# )r'   rQ   r  r*   rA   r+   )r,   s   "r-   r.   r.   	  s     6 6 6T 6r/   c           	        \         pR  R lp\        WR7      p. pVP                  RRP                  V4      ,           R,           4       VP                  RRP                  R.\	        V4      ,          4      ,           R,           4       V F  p\	        V4      \	        V4      8  d,   VR.\	        V4      \	        V4      ,
          ,          ,           pM'\	        V4      \	        V4      8  d   VR\	        V4       pVP                  RRP                  V4      ,           R,           4       K  	  \        \        RP                  V4      R,           4       R# )c                   V ^8  d   QhRR/# )r'   r"  r  r+   )r,   s   "r-   r.   $write_markdown.<locals>.__annotate__	  s      Y r/   c                    V ^ ,          p ^ \        \        P                  ! RRV4      4      3#   \         d    ^T3u # i ; i)r{   z\Dr   )r`   r:   r;   rb   )r"  r4   s   & r-   	style_key!write_markdown.<locals>.style_key	  sE    aD	s266%Q/011 	q6M	s   ". A A )r  r6   z---r   Nrq   )
MD_HEADERSr  r  r  r  rJ   OUT_MD)rQ   r  r  sorted_rows	out_linesr"  s   &     r-   write_markdownr  	  s    G -KIS388G,,s23S388UGc'l$:;;cABq6CL RDCL3q6122AVc'l"-3w< Asxx{*S01  9-45r/   c                   V ^8  d   QhRR/# r  r+   )r,   s   "r-   r.   r.   	  s      9 r/   c                 d   \         P                  4       '       g   . # . p \         P                  R RR7      P                  4        Fh  pVP	                  4       pV'       g   K  \
        P                  ! RV4      pV'       g   K>  VP                  ^ 4      pW09  g   KW  V P                  V4       Kj  	  V # )rC   r9   )rE   r  zhttps?://\S+)	rs   rY   r_   r  r<   r:   r  r  r  )r]  rn   r  r  s       r-   _read_errors_urlsr  	  s    	D$$gi$HSSUzz|IIot,1
A}A V Kr/   c                    V ^8  d   QhRRRR/# )r'   r]  r  r*   rA   r+   )r,   s   "r-   r.   r.   0	  s     6 6y 6T 6r/   c                    V '       g    \         P                  R R7       R# V  Uu. uF  pR\         RV 2NK  	  pp\	        \         RP                  V4      R,           4       R#   \         d     R# i ; iu upi )Tr#  NFAILED x: rq   )rs   r0  rb   MAX_FAILS_PER_LINKrJ   r  )r]  r  r  s   &  r-   _rewrite_errors_filer  0	  sw    	. 	;?@4aw)*#aS14E@TYYu-45	  		 As   A% A7%A43A4c                   V ^8  d   QhRR/# r   r+   )r,   s   "r-   r.   r.   ;	  s     d dd dr/   c                    \        4       p \        V P                  R . 4      4      p\        V P                  R/ 4      4      p\	        V P                  R. 4      4      p\        V P                  R\        P                  ! 4       4      4      p\	        V P                  R. 4      4      pV'       g   \        4       p\        '       d   \        V 4      pV'       dm   \        V P                  R . 4      4      p\        V P                  R/ 4      4      p\	        V P                  R. 4      4      p\	        V P                  R. 4      4      pRpRp \        V \        R\        R7      p\        V4       V P                  R4      ;'       g    . p	V P                  R	4      ;'       g    / p
\        '       g	   V	'       g   \        ^VR
7      pVP!                  ^ RR7       \#        V4      w  rV'       d   Tp	M\$        P&                  P)                  R4       V'       d   Tp
 \*        '       d>   V
'       d6   \-        W:4      pV'       d#   W0R&   WR	&   \/        V 4       \1        RV R24       YR&   \*        '       d   YR	&   \/        T 4       \*        '       dd   V
'       g\    \#        V4      w  rWR	&   \/        V 4        V
'       d6   \-        W:4      pV'       d#   W0R&   WR	&   \/        V 4       \1        RV R24       \4        '       dU   \7        V 4      pV P                  R4      '       g   RV R&   \/        V 4       V'       d   \1        RV R24       \/        V 4       \8        P;                  4       '       dQ   \8        P;                  4       pT	;'       g    .  Uu. uF  p\=        V4      V8X  g   K  VNK  	  p	pWR&   \/        V 4       \?        V	4      pV^ 8X  d   \A        R4      hV'       d   VPC                  4        \        VVR
7      p\?        V4      pVP!                  VRR7       \E        V4       UUu/ uF+  w  ppV'       g   K  V^ ,          '       g   K!  V^ ,          VbK-  	  pppV	 EF  pVV9   d   V^,          pVP!                  VRR7       K)  Rp  \G        VV4      pV^ ,          '       g   \A        RV 24      hV^,          '       g   \A        RV^ ,           RV R24      h\*        '       dA   V
'       d9   \I        V
P                  V^ ,          R4      4      pV'       d   \K        V4      V^&   \?        V4      ^8  d   V^,          '       g   \K        V4      V^&    \?        V4      ^8  d>   V^,          '       d/   \K        \M        V^ ,          V^,          V^,          4      4      V^&   T^ ,          T9   d   TTTT^ ,          ,          &   M&\?        T4      TT^ ,          &   TPO                  T4       TPQ                  T4       \S        T4      T R &   Y0R&   Y R&   YPR&   \/        T 4       \U        T4       Rp T^,          pTP!                  TT'       d   R#MR$R7       EK  	  \U        V4       VP!                  VR%R7       \h        '       Ed   V'       Ed   \$        P&                  P)                  R&\?        V4       R'24       . p V'       d    VPg                  4        \        V \        R(\        \j        R)7      p\        V4       \E        \	        V4      ^R*7       F  w  ppRp\m        \n        4       F  p \G        VVRR+7      pV'       do   V^ ,          '       d^   V^,          '       dM   V^ ,          V9   d   VVVV^ ,          ,          &   M&\?        V4      VV^ ,          &   VPO                  V4       Rp MK  K  K  	  V'       d5   \$        P&                  P)                  R.V R \?        V4       R/V R"24       K  VPO                  V4       K  	  TpW0R&   WPR&   \/        V 4       \U        V4       \s        V4       \$        P&                  P)                  R1\t         R"24       \v        Py                  4       '       d(   \$        P&                  P)                  R2\v         R"24        V'       d   VPC                  4         T'       d   TPg                  4        R# R#   \2         d     EL|i ; i  \2         d    T
;'       g    / p
 ELJi ; i  \2         d     ELi ; iu upi u uppi   \2         d     ELi ; i  \2         Ed   p\W        TP                  T^ 4      4      ^,           TT&   \S        T4      T R &   Y0R&   Y R&   YPR&   \/        T 4       TT,          \X        8  df   TT9  d*   TPO                  T4       \[        RTT,           RT 24       TPQ                  T4       \S        T4      T R &   YPR&   \/        T 4        Rp?EK  \$        P&                  P)                  RTT,           R \X         R!T R"24       \$        P&                  P)                  RP]                  \^        P`                  ! \c        T4      TTPd                  4      4      R",           4        T'       d   TPg                  4        M  \2         d     Mi ; i\        T \        R\        R7      p\        T4        Rp?EKY  Rp?ii ; i  \2         d     EL"i ; i  \2         d@     TP                  R,4       M  \2         d     Mi ; i\        Pp                  ! R-4        EK  i ; i  \2         d   p\$        P&                  P)                  R04       \$        P&                  P)                  RP]                  \^        P`                  ! \c        T4      TTPd                  4      4      R",           4        Rp?ELRp?ii ; i  \2         d     EL:i ; i  \2         d     R# i ; i   T'       d   TPC                  4        M  \2         d     Mi ; i T'       d   TPg                  4        i i   \2         d     i i ; i; i)3rP   rT   rQ   rV   rU   NrP  )rU  rO  rQ  rO   rR   )r   rV   zRefreshing product links...)r   zN[warn] Link refresh discovered 0 links; falling back to checkpoint link list.
z)Applied preferred International names to z existing rowsrS   Tz
Re-queued z links due to missing URL/Imagez{No product links discovered from catalog page. Login/navigation likely succeeded but product link extraction found nothing.startingz	(resumed)Fz$Style number parsed empty for link: zName parsed empty for style r   )r   r  r  z
Error scraping link (attempt r   z): rq   rr  skippeddonez
Starting salvage pass for z failed links...
normal)rU  rO  rQ  rR  )r  )r  ri  g      ?z	Salvaged z: z$
Salvage pass encountered an error:
z
DONE. Wrote: z#Some links failed repeatedly; see: )=rf   r  ra   r  r  r   r\   r  AUTO_REPAIR_BAD_ROWSr  rf  RUN_HEADLESSBLOCK_IMAGES_IN_BROWSERrl  REFRESH_PRODUCT_LINKS_EACH_RUNr   r   r  r   r  rt   r  r  rl   printrb   AUTO_REPAIR_MISSING_MEDIAr  r    r<   r  r  r(  r   r  r  r  r=   r  r  r  r  r  r`   r  rw   r  	tracebackformat_exceptiontype__traceback__quitENABLE_SALVAGE_PASSSALVAGE_PAGE_LOAD_TIMEOUTr  SALVAGE_MAX_TRIES_PER_LINKrp  r  r  rs   rY   )rG  rP   rT   rQ   rV   rU   repairedreporterre  rO   rR   fresh_linksfresh_preferred_namesnupdr  rmwantr  r   	processedr	  r"  style_to_idxr  rr  r  r  rL  	remainingsalvageds                                 r-   mainr  ;	  s
   		BrvvlB78J"&rvvmR'@"AK !34DrvvlDIIK89J#'/BB(G#H-/
 -b1RVVL"56JrvvmR89Kvr*+D $RVV,?%D E ,0HFI!&0	
 	F#/552*,&&1B*C*I*Ir))'aJGHOOA$AOB1Fv1N.K +

  e %"7	55/0GD%)6
0?,-'+ I$~^_ #011(7$%B .-o8%:6%B"(7$%#
	"0GD%)6
0?,-'+ I$~^_ %$%b)B66-..*.&'#
2$&EFG# ##%D)6)<)<")<c)<1AWXYAZ^bAbQQ)<Mc"/BM"A:_  NN#%JG
O		
3 -6dOJODAqqQqTT!aOJ!Dz!Q		<BQ0(6Cq66*-QRVQW+XYYq66*-I#a&QSTXSYYZ+[\\ :9o2?3F3Fs1vr3RS%3D%9CF 3x!|CFF!/!5As8a<CFF%34KCPQFTWXYTZ\_`a\b4c%dCF
 1v-58\#a&12/24ySV,C(NN4('-j'9B|$!%vJ(3}%.?*+#B'"4(BP NIOOIRTYOHy "~ 	tF+ #4#4JJ;C@Q<R;SSefg#%I,j ')'/!8&? $F+(.?)@JGAt$H"#=>,"04"PC"s1vv#a&&#&q6\#9ADDc!f)=$>;>t9LQ$8$(KK$4+/ % 39vs ?$  

((9QCq=N9O8PPRSWRXXZ)[\!((./  K2 %.! vJ&7"#B4  !23

?6("56JJB:,bQR	 	 I    8"1"7"7R8    d$ K@ % . ! $0(+KOOD!,D(E(IK%'-j'9B|$!%vJ(3}%.?*+#B'"4(,>>'88-44T:-D8I7J#dV.TU"t,+1*+=<(2C./'+JJ$$9+d:K9LAN`Maadeidjjlm JJ$$RWWY-G-GQQRTUTcTc-d%ehl%lm!"KKM$ *!-+2%<	F (//I$0l % 0  ) ,% &

= 9#, % $% JJsOO,  j

  !IJ

  )C)CDGQPQP_P_)`!adh!hiij$  		
  			  			  		s  9o? o? o? +o? 3Ao? o? c !c )5c *o? 	o? c /c< 75c< ,o? 89o? 23o? &o? o? dd%5o? A	o? $d6dd1o? 7d+?$d+$d+,Ad+d+d1.dBd+*o? Ao? 
.o? 9m k& Am *$k8k8A
k8(m :Am A)o? /'o? o 0o- 8o- co? co? c92c95o? 8c99o? <do? 
do? d($d+'d((d++k#7B:k1o? 8Bkj$#k$j2/k1j22%ko? k##o? &k51m 4k55m 8mlml$	!m#l$	$m=m mm oA=oo? oo? o*)o*-o<;o<?qpqp(%q'p((q,q4qqqqqq__main__)z(https://order.redwingshoes.com/other-ppez0https://order.redwingshoes.com/accessories-beltsz/https://order.redwingshoes.com/accessories-capsz8https://order.redwingshoes.com/accessories-care-productsz/https://order.redwingshoes.com/accessories-gearz2https://order.redwingshoes.com/accessories-insolesz0https://order.redwingshoes.com/accessories-lacesz9https://order.redwingshoes.com/accessories-safety-glassesz1https://order.redwingshoes.com/accessories-shirtsz3https://order.redwingshoes.com/accessories-slippersz0https://order.redwingshoes.com/accessories-socksz6https://order.redwingshoes.com/accessories-work-gloves)z
/other-ppez/accessories->   r   yonyestrue)r  r  URLImagezShort DescriptionDescriptionzProduct Details)rA  g?)T)   )P   )x   r   )__conditional_annotations__r   
__future__r   r]   r   r  r:   r   r1  r   r.  r\   r  rq  rq  r  r  html.parserr   r*  pathlibr   typingr   r   r   r   r	   urllib.parser
   r   r   urllib.requestr   r   bs4r   seleniumr   selenium.common.exceptionsr   r   selenium.webdriver.common.byr   selenium.webdriver.common.keysr   "selenium.webdriver.firefox.optionsr   "selenium.webdriver.firefox.servicer   r_  selenium.webdriver.supportr   rE  selenium.webdriver.support.uir   r[   r  rY  r  r   ra   r<   r   r   r  r  r  r    __annotations____file__r@  r?  r   r  rX   rs   r  r  r  r  r  r  r  r  r  r  PAGE_LOAD_TIMEOUTrc  NAV_SETTLE_SECONDSro  r\  r  r  r  rZ   r%   r1   r=   rJ   rf   rl   rw   r}   r   r   r   r   r  rE  rN  rf  rl  ru  r  r  r  r  r  r  r  r  r  r'  r=  rI  rQ  r`  ru  r  r+  r  r  r  r  r  r  r  r  r  r  r  r  r  r*  r3  r:  rA  rE  rg  rk  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r   )r  s   @r-   <module>r     ss  +Z #  	  	   
       "   3 3 4 4 +
   K + / 6 H @ 7 #1o $	9  JJNN#56TT:T[[] JJNN#56JJ:JQQS zz~~mS1779??AEdd!#0BC!H!N!N!P!V!V!X\{!{  
C >!!#**	4	4FF
AA
 
  +  &+ "  3S9??AGGIMll **..S %'%%'45  h
J !     #         ::>>"4b9??AIIT <
~I&
4G GZ#L!0AH!
H5 &	5
 5 /5 !%5 5p, %&CN"2$J $4
3#r"JCL=@aH0&RL^ 1FC*H.BL41h"<&~$R"&P-d62	&9!.<@1f8v",,9$(T$ $ $ $N/:1MU 1Md64"6dN	 zF r/   