?i,mUdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZddlZddlmZddlmZmZmZmZmZddlmZmZmZddl m!Z!m"Z"dd l#m$Z$dd l%m&Z&dd l'm(Z(m)Z)dd l*m+Z+dd l,m-Z-ddl.m/Z/ddl0m1Z2ddl3m4Z5ddl6m7Z7dgZ8e8dZ9dZ:dZ;ejxj{ddZ>ejxj{ddZ?ejxj{ddjjdvZBejxj{ddjjdvZCdZDdeEd<eeFjjZIeId z ZJeId!z ZKeId"z ZLd#ZMd$ZNd%ZOd&ZPgd'ZQeReQZSd$ZTd(ZUd)ZVd*ZWd+ZXd$ZYd$ZZd,Z[d-Z\d.Z]ejxj{d/djxsdZ^dwd0Z_dxd1Z`dyd2Zadzd3Zbd{d4Zcd|d5Zdd}d6Zed~d7ZfGd8d9Zgdd:Zhdd;Zidd<Zjdd=Zkdd>Zl dd?d$eUdd@ ddAZmddBZnddCZoeUeWd-f ddDZpddEZqdddFZrGdGdHeZsddIZtddJZuddKZvddLZwdMZxddNZyddOZzddPZ{ddQZ|ddRZ}dddSZ~ddTZddUZdddVZdxdWZddXZddYZddZZdd[Zdd\Zdd]Zdd^Zdd_Zdd`ZddaZddbZddcZdddZdddeZddfZddgZdhZddiZddjZddkZddlZddddm ddnZdddoZd&dpddqZddrZddsZddtZdduZedvk(reyy)a RW_Site_Scraper-Orders_Page.py ============================== Scrapes Red Wing footwear catalog: - https://order.redwingshoes.com/footwear-rwbr Test mode: - Set ONLY_STYLE at the top of this file to a style number (e.g. "400") to scrape just that single product. Outputs (next to this script): - RW_Orders_Site_Scrape.md - RW_Site_Scraper_v2_checkpoint.json (resume state) - RW_Site_Scraper_v2_errors.txt (links that failed repeatedly) Stability features: - `safe_get()` uses short timeouts + window.stop() so Selenium doesn't hang forever - HTTP fallback (no Selenium) for the few product pages that still time out - Optional salvage pass at the end to retry hard failures (HTTP-first) Cross-platform (Windows + Linux Cinnamon): - Headless Firefox (default) - Geckodriver resolution "like Parts_Auto" (explicit Service path; no Selenium Manager): 1) GECKODRIVER_PATH env var (file or directory) 2) geckodriver(.exe) on PATH 3) auto-download geckodriver (GitHub releases) into a user cache dir 4) (optional) if double-click/no terminal and Tk is available, prompt to pick geckodriver Feature columns are 1/0 (not Yes/No). Includes Brand (string) + brand family flags (Red Wing / Irish Setter / Worx). Dependencies: - Python 3.9+ - Firefox installed - Selenium installed: Linux Mint/Ubuntu: sudo apt install -y python3-selenium Windows: python -m pip install selenium Notes for Linux Mint PEP 668: - Prefer `python3-selenium` from apt (as above). - This script does NOT require webdriver-manager. ) annotationsN) HTMLParser)Path)DictListOptionalSetTuple)parse_qsurljoinurlparse)Requesturlopen) BeautifulSoup) webdriver)TimeoutExceptionWebDriverException)By)Keys)Options)Service)expected_conditions) WebDriverWaitz,https://order.redwingshoes.com/footwear-rwbrzorder.redwingshoes.comz/footwear-rwbrRW_SITE_USERNAMEzrwss614@redwingshoes.comRW_SITE_PASSWORDzWelcomeBack99! RW_HEADLESS1>ryonyestrueRW_REFRESH_LINKSstr ONLY_STYLEzRW_Orders_Site_Scrape.mdz"RW_Site_Scraper_v2_checkpoint.jsonzRW_Site_Scraper_v2_errors.txtTRED WING FOR BUSINESSF)'Style #NameURLImageBrandMaleFemaleRed Wing Irish SetterWorxz Safety Toez Steel ToezNon-Metallic Toez Aluminum ToezMetatarsal GuardzSoft Toe Waterproof InsulationzSlip ResistantzElectrical HazardzPuncture ResistantzStatic DissipativezAnkle ProtectionuBOA® Lacing System Defined HeelzAll Leather UpperzOxford/AthleticChukkaHiker5"6"7"8"9"10"11"12"z Built in USAz Made in USA-g?)333333?g?Z GECKODRIVER_PATHc|rdSdS)Nr0)vs ,Boot_Features/RW_Site_Scraper-Orders_Page.pyb01rKs3c||xsdjdd}tjdd|j}|S)Nr#|z\|\s+ )replaceresubstrip)ss rJmd_escape_cellrVs8 b#u%A vsA$$&A HrLc|j|jdz}|j|d|j|y)Nz.tmputf-8encoding) with_suffixsuffix write_textrQ)pathtexttmps rJ atomic_writeras7   4;;/ 0CNN4'N*KKrLc 4tjs(ttgggidigt jdd S t j tjd}t|jdd}|tk7rR tjtjd|dttgggit jdd S|jdt|jd t|jd g|jd g|jd g|jdi|jdg|jdt j|jdd|S#t$rYwxYw#t$rd tjtjdn#t$rYnwxYwttgggidigt jdd cYSwxYw)NF) version catalog_urls product_links done_linksrowspreferred_namesmedia_repair_done fail_countshard_failed_links started_atgeckodriver_pathrXrYrcrz.json.vz.bak)rcrdrerfrgrjrlrmrdrerfrgrjrkrlrmz .json.corrupt) CHECKPOINTexistsCHECKPOINT_VERSION CATALOG_URLStimejsonloads read_textintgetrQr[ Exception setdefault)dataold_vers rJload_checkpointr|s    )(!!&!#))+ $  - zz*...@Adhhy!,- ( ( "":#9#9GG9D:Q#RS. ,!# !"iik$(    #56  5 ,  b) #  r* +R0  diik2 *D1 +  ,      z55oF G   *(!!&!#))+ $   saA F* ,F8$F*B=F* F'$F*&F''F** H4(GH G)&H(G))+HHcPtttj|ddy)NrCT)indent sort_keys)rarnrsdumps)rzs rJsave_checkpointrsTZZQ$GHrLcttdd5}|j|jdzdddy#1swYyxYw)NarXrY )open ERRORS_TXTwriterstrip)linefs rJwrite_errors_liners< j# 0&A  $%&&&s #?Ac, ddl}y#t$rYywxYw)NrTF)tkinterrx)rs rJ _can_use_tkrs s  ctjjs tsy ddl}ddlm}m}|j}|j|jdd|jdd|j|}|j|xsdj}|r|SdS#t$rYywxYw) Nr) filedialog messageboxz-topmostTRW Site ScraperzCould not find geckodriver automatically. Please select the geckodriver executable. Windows: geckodriver.exe Linux: geckodriver)titler#)sysstdoutisattyrrrrTkwithdraw attributesshowinfoaskopenfilenamedestroyrTrx)rtkrrrootr^s rJ _tk_pick_filers zz+-2uuw   D)  ; )))6  !!#t%% sB B75B77 CCc*eZdZdZddZdddZd dZy) ProgressReporterzDTTY progress bar, or Tk window if launched by double-click (no TTY).ctt|d|_||_tj j |_d|_d|_ d|_ d|_ |jsEtr9 ddl }ddl m}|j|_ |jj!d|jj#d|jj%dd|j'|jdd |_ |jj)d d d |j+|j|jd|_ |jj)d dd|_|jj-|jj/yyy#t0$r d|_YywxYw)NFr)ttkr620x150z Starting...w)r_anchorx )r)fillpadxpadyiD)maximumlength)rrD)rrT)maxrvtotalrlrrruse_ttygui_root_label_pbarrrrrrgeometry resizableLabelpack Progressbarupdate_idletasksupdaterx)selfrrlrrs rJ__init__zProgressReporter.__init__$sSUQ' $zz((*     +- !$'UUW    !23 ##I. $$UE2 hhtzz chR    c A __TZZTW_X  Rg6 ++- !!#%#0 & !  !s:D7F44GGc >tdtt||j}|jr|j r|d|jd|j }|jr|jj||jr||jd<|j j|j jyd}||jz }tt||z}d|zd||z zz}ttj|jz d }||z } | d kDr|j|z | z nd } d |d |d|jd|dzddt| d }|r|d|z }tj j#d|ddztj j%||jk(r>tj j#dtj j%yy)Nr/z )r_value #-g-C6?g& .>[z]  (dz5.1fz%) ETA rU r)rminrvrrrrTrconfigrrrroundrrrlrrrflush) rcurrentnotemsgwidthfracfilledbarelapsedrateetas rJrzProgressReporter.updateCsaS\4::67 88 IQtzzl"TF399;C{{ """,zz&- 7# JJ ' ' ) JJ    #U4%<()FlSEFN33diikDOO3V< /3d{tzzG#t+#b 4::,bc$ws3xjPQR  Rv; C DS )*  djj JJ  T " JJ    !rLc|jr)|jr |jjyyy#t$rYywxYwN)rrrrx)rs rJclosezProgressReporter.closecs? 88   ""$#8  s7 AAN)rrvrlfloatr#)rrvrr$returnNonerr)__name__ __module__ __qualname____doc__rrrrHrLrJrr!sN!>@rLrctjdk(rntjjdxs>tjjdxst t j }t |dz }nOtjjdxs t t j dz }t |dz }|jdd |S) Nnt LOCALAPPDATAAPPDATARW_Site_ScraperXDG_CACHE_HOMEz.cacherw_site_scraperTparentsexist_ok)osnameenvironrwr$rhomemkdir)baseds rJ _cache_dirrns ww$zz~~n-^ 1J^cRVR[R[R]N^ J* *zz~~./N3tyy{X7M3N J* *GGD4G( HrLcftrctt}|jr t|S|j r)dD]$}||z }|jst|cSdD](}t |z }|jst|cSt jdk7rXtjdz tjdz dz fD]$}|dz }|jst|cStjdxstjd}|r|Sy)N)geckodriver.exe geckodriverrbinz.localrr) rEris_filer$is_dirBASE_DIRrrrshutilwhich)prcandrs rJ_resolve_from_env_or_pathrys ! " 99;q6M 88:: %4x<<>t9$ % 3$ <<>t9   ww$))+%tyy{X'='EF !A}$D||~4y  !  ]#Fv||4E'FA rLc@tjj}tjj}|j dr#dtj dv}|rddfSddfS|j dr d|vsd |vry y |d k(r d |vsd|vry yy )zO Returns (asset_contains, archive_type) matching geckodriver releases. win64rwin64win32ziplinuxaarch64arm64)z linux-aarch64tar.gz)linux64r darwin)z macos-aarch64r )macosr )rplatformlowermachine startswith architecture)sysplatmachis_64s rJ_platform_asset_keyrsll  "G     # # %D% --/22 u55gu55'"  4.$( d?i4/." rLct\}}d}t|ddi}t|d5}tj|j j dd }d d d jd g}d }d } |D]?} | jd d } || vs| j|s,| jd}| } n|std|d|d|| z } t|ddi} t| d5}t| d5}tj||d d d d d d tjdk(rdnd}||z }|dk(rqtj | d5}|j#D]>}|j|s|j%||||z }|j'|nd d d nt)j| d5}|j+D]d}|jjd|zs|j|k(s1|j%||||jz }|j'|nd d d  | j-dtjdk7ri tj0|}tj2||j4t0j6zt0j8zt0j:z|j=s td|S#1swYxYw#1swYxYw#1swYxYw#1swYxYw#1swYxYw#t.$rYwxYw#t.$rYvwxYw)zh Downloads and extracts latest geckodriver into dest_dir. Returns path to extracted driver. z@https://api.github.com/repos/mozilla/geckodriver/releases/latest User-AgentzRW_Site_Scraper/1.0headersr@timeoutrXrQerrorsNassetsrr#browser_download_urlz'Could not find a geckodriver asset for rz).<wbrrrr r)r^zr:gzrT missing_okz5Download succeeded but geckodriver was not extracted.)rrrrsrtreaddecoderwendswith RuntimeErrorrr copyfileobjrrzipfileZipFilenamelistextractrQtarfile getmembersunlinkrxstatchmodst_modeS_IXUSRS_IXGRPS_IXOTHro)dest_dir asset_key archive_typeapireqr&rzr"dl_urldl_namerr archive_pathreq2r driver_nameextracted_pathzmemberrtsts rJ_download_latest_geckodriverrJs5 23I| LC # .CD EC b !FQzz!&&(//')/DEFXXh #F FG uuVR   |!<UU12FG   DYKrR^Q__abccg%L 6L2G#H ID r "!alD)A!Q1a !!(*ww$#MK +Nu __\3 / 1**, ??;/IIf8I4 6)AIIn-    \\, / 1,,. ;;''k(9:fkk[>XIIf8I4 6;;.AIIn-    t, ww$ (B HH^RZZ$,,%>%MPTP\P\%\ ]  "RSS qFF$!!!!           sy5L8 LL L#%L' ,L'AL36L3L?;A(ML  L LL$'L03L<? M  M  MMct}|rt|jr|S|jdxsdj }|rt|jr|S t }t |}t||d<t|t|S#t$rP}td}|r0t|jr||d<t||cYd}~Std|d}~wwxYw)Nrmr#z$Select geckodriver / geckodriver.exezUnable to locate or install geckodriver. Fix options: - Put geckodriver on PATH - OR set GECKODRIVER_PATH to the full driver path - OR install via package manager (Linux often: sudo apt install firefox-geckodriver) Underlying error: ) rrrorwrTrrJr$rrxrr,)ckrsaveddest driver_pathepickeds rJensure_geckodriverrRs!#AT!W^^ VV& ' -2 4 4 6E e##%  |248 !$[!1 ;  EF d6l))+%+B! " B M ! "#  %   s$)8B"" C;+7C6"C;(C66C;eager)page_load_strategy block_imagespage_load_timeout user_agentc>t|}t}|r|jdtr|jd|r|j d||j dd|j dd|j dd |j d d|j d d|j d d|j d d|j dd|j dd|j dd|j dd |j dd |r|j dd|r|j d|t |}tj||} | j|| jt| S)aJStart Firefox using an explicit geckodriver path (no Selenium Manager). Args: page_load_strategy: "eager" returns after DOMContentLoaded. "normal" waits for full load. block_images: If True, blocks images to reduce load stalls/timeouts. page_load_timeout: Seconds for Selenium navigation timeout. z -headlessz-privatepageLoadStrategyzdom.webnotifications.enabledFzmedia.volume_scalez0.0z!browser.privatebrowsing.autostartTznetwork.http.http3.enablezbrowser.cache.disk.enablezbrowser.cache.memory.enablezbrowser.cache.offline.enableznetwork.http.use-cachezplaces.history.enabledzsignon.rememberSignonsz"privacy.trackingprotection.enabledz)privacy.trackingprotection.pbmode.enabledzpermissions.default.imagerCzgeneral.useragent.override)executable_path)serviceoptions) rRr add_argumentPRIVATE_BROWSER_MODEset_capabilityset_preferenceFirefoxServicerFirefoxset_page_load_timeoutset_script_timeoutSCRIPT_TIMEOUT) rLheadlessrTrUrVrWgeckor\r[drivers rJ create_driverris r "EiG[)Z(13EF 95A /7 >E 6> 6> 8%@ 95A 3U; 3U; 3U; ?F FM:A>;ZHU3G   w @F   !23 n- MrLc |j |jd|jdy#t$rY/wxYw#t$rYywxYw)z;Clear cookies/storage for a clean private session baseline. about:blankz} try { localStorage.clear(); } catch (e) {} try { sessionStorage.clear(); } catch (e) {} N)delete_all_cookiesrxrwexecute_script)rhs rJreset_browser_staternQs^ !!#   =!         s 6"A AA AAct\}} tjtj||y#t $rtj|YywxYwr)REQUEST_DELAY_RANGErrsleeprandomuniformrx)lohis rJ _polite_delayrvgsA FB 6>>"b)*  2s)5AAcd}td|dzD]} |j|t t|dd}|~t |dr3t |j drt |dz|j _t |dr3t |jdrt |dz|j_|j|tj| |jtjd y|r|y#t$rYZwxYw#t$rYywxYw#t $ry}|} |j#d n#t$rYnwxYwtj| |jtjd Yd}~y#t$rYnwxYwYd}~nd}~wt$$r }|}Yd}~nd}~wwxYw |jd n#t$rYnwxYwtjd |z) zNavigate without getting stuck on pages that never fully finish loading. - Uses page_load_timeout - On timeout, calls window.stop() and continues if the DOM exists - Retries a couple times with light cleanup NrrCcommand_executor_client_configrrD_connbodyzwindow.stop();rkg?)rangercrvgetattrhasattrryrvrrzrxrwrrrq find_elementrTAG_NAMErrmr) rhurlrsettle max_trieslast_excattemptcerPs rJsafe_getros)-HIM*1#)   ( ( 1 O V%7>>r#34ARART]9^47L24E))1r7+)0L+.w<"+<( JJsO JJv  ##BKK8 51#fK      H %%&67   JJv  ##BKK8  ! H   JJ} %    4'>"sD-B D?&D-& D DD-DD- D*&D-)D**D-- G6F*9E  F* EF*EF*0 F F"F*!F""F** G6F==GG G$#G$cg}|fD]}||vs|j|d|vr*|jddd}||vr|j||S|jddd}||vr|j||S)zOReturn a small set of URL variants (www/non-www) to dodge occasional redirects.z//www.z//r)appendrQ)routsuu2s rJ _url_variantsrsDU D= KKN3 [[4 + T> KKO K[[x + T> KKO KrLct|dddd}t||5}|j}dddjdd S#1swYxYw) Nz_Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zen-US,en;q=0.9)rAcceptzAccept-LanguagerrrXrQr )rrr)r*)rrr?r&rzs rJ _fetch_htmlrs`  {W/  C g &!vvx ;;wy; 11s A  Ac<eZdZdZhdZfdZdZdZdZxZ S)_VisibleTextExtractorz;Very small HTML->visible text extractor (no external deps).>rbrh1h2h3h4h5h6litdthtrdivfooterheaderarticlesectionc0t|g|_yr)superrparts)r __class__s rJrz_VisibleTextExtractor.__init__s  " rLct|j|jvr|jjdyyNrr _BLOCK_TAGSrr)rtagattrss rJhandle_starttagz%_VisibleTextExtractor.handle_starttag- 99;$** * JJ  d # +rLct|j|jvr|jjdyyrr)rrs rJ handle_endtagz#_VisibleTextExtractor.handle_endtagrrLcb|r-|jr|jj|yyyr)rTrr)rrzs rJ handle_dataz!_VisibleTextExtractor.handle_datas& DJJL JJ  d #!4rL) rrrrrrrrr __classcell__)rs@rJrrs!EK #$$$rLrc4t} |j|dj|j}t j |}tjdd|}tjdd|}|jS#t$rYywxYw)Nr#z\n{3,}z z[\t\r]+rP) rfeedrxjoinrhtml_libunescaperRrSrT)htmlrraws rJ _html_to_textrsA  t  ''!'' C   C C &&FC (C &&S# &C 99;    sB BBc>tjd|tjtjz}|sy|j d}tj dd|}t j|}tj dd|j}|S)Nz]*>(.*?)flagsr#r<[^>]+>rPrO) rRsearch IGNORECASEDOTALLgrouprSrrrT)rminners rJ_extract_first_h1rsw *D  8QRA  GGAJE FF:sE *E   e $E FF63 & , , .E LrLcdhhdd d}d fd }tjd|tjtjz}|r!||j d}||r|Stj d|tjtjzD]%}||j d}||s#|cSt |S) aExtract a likely product title from HTML. Many Red Wing product pages place the actual product name in

, while

can be a site/banner header (e.g., 'RED WING FOR BUSINESS'). We therefore try meaningful

first, then fall back to

. r'> SIZE & FITRELATED PRODUCTSCARESIZINGDETAILSREVIEWSFEATURES TECHNOLOGYSPECIFICATIONSctjdd|}tj|}tjdd|j }|S)NrrPrO)rRrSrrrT)rs rJ_cleanz&_extract_first_heading.._cleansDz3.!!%(vsE*002 rLc|xsdj}|sy|j}|vs|vryt|dkrytjd|syy)Nr#Fr&z[A-Z]T)rTupperlenrRr)rUupBADSTOPs rJ_okz#_extract_first_heading.._oksV W"OO  WWY 9d  r7a<yy2&rLz]*class=['\"][^'\"]*prTitle[^'\"]*['\"][^>]*>\s*Name\s*\s*]*class=['\"][^'\"]*prValue[^'\"]*['\"][^>]*>(.*?)rrz]*>(.*?)

)rr$rr$rUr$rbool)rRrrrrfinditerr)rrrmnamerUmh3rrs @@rJ_extract_first_headingr s # #C D   II I mmbii'  E  5;;q> " q6H{{3TQSQZQZAZ[ 399Q<  q6H T ""rLc|xsdj}|sy|j}|jds"|jds|jdryd|vry t|}|jxsdjj d}t |jxsd}d |vrQtjd |r:d D]4}|j|s |j|js4yyyd |vrtjd |rytjd|ryy#t$r|}i}YwxYw)Nr#F javascript:zmailto:ztel:z /safety-boot/Tr /footwear-z.*/footwear-[^/]+style styleNumberskuitempid productIdz /product/z\d{3,6}z/\d{3,6}\.html$) rTrrr r^rr queryrxrR fullmatchrwr)hreflowparsedr^rkeys rJ_is_product_linkrCs4 JB   D  **,C ~~m$y(AS^^TZE[#$ !r((*11#6+, t <<,d 3R 99S>UYYsyy{%; dryyT: yy#T* + s"AD;; E  E cg}t} |jtjd}|D][} |j dxsdj }|r||vr.t|s:|j||j|] |jtjd}|D][} |j dxsdj }|r||vr.t|s:|j||j|] |s |jtjdD][} |j dxsdj }|r||vr.t|s:|j||j|] |S|S#t $rd}YNwxYw#t $rY&wxYw#t $rd}YwxYw#t $rYwxYw#t $rd}YwxYw#t $rY|SwxYw)uLocate product anchors on the catalog page. The Red Wing wholesale catalog renders product tiles with specific CSS classes. Each tile (
  • ) contains two links pointing at the same product detail page: one wrapping the image and one wrapping the product name. Both have an href that ends in ".html" with the numeric style number. On the current site the anchor for the name has the classes ``c-product-tile__pdp-link js-product-name``, while the image link has no special class but is still within the tile. This helper first attempts to extract anchors using those specific selectors and falls back to scanning all anchors when necessary. Returns a list of WebElement objects (anchors) with unique hrefs. Hli.js-product-grid-item a.c-product-tile__pdp-link.js-product-name[href]rr#(li.js-product-grid-item a[href$='.html']z //a[@href]) set find_elementsr CSS_SELECTOR get_attributerTrxrraddXPATH)rhanchorsseenelementsrr elements2s rJ_find_product_anchorsrhsGUD '' OO V  A /52<<>44<%q!  (( OOG  A /52<<>44<%q!   ))"((LA #OOF39r@@BDtt|#D)NN1%HHTN # N7NS            !D   N s%F)#FF)*$F)%G 6#F9G ,$G #G*8#GG*.$G* F&"F)%F&&F)) F65F69 GG GG GG G'$G*&G''G** G76G7ct |jxst} |jxsd}|sgSt } t |d}|j dD]9}|jdxsdj}|s)|j|;|j dD]9}|jdxsdj}|s)|j|;|jddD]9}|jdxsdj}|s)|j|; tj|}gd }|D]d} tj| |tj D]9} | j!d xsdj} | s)|j| ;fg} t } |D]A}t#||}t%|s|| vr | j|| j'|C| S#t$r t}YwxYw#t$rd}YwxYw#t$rY wxYw) zIFallback link discovery when clickable anchors are sparse or JS-rendered.r# html.parserrrrrT)r)zhttps?://[^\s\"'<>]+z2/(?:footwear-rwbr|safety-boot|product)/[^\s\"'<>]+z"/\d{3,6}\.html(?:[?#][^\s\"'<>]*)?rr) current_urlCATALOG_URL_PRIMARYrx page_sourcerrselectrwrTrfind_allrrrRrrrr rr)rhbase_urlr candidatessouprrblobpatternspatrroutrrabs_urls rJ'_extract_product_links_from_page_sourcers/'%%<)<!!'R  5J T=1gh %AEE&M'R..0Dt$ % GH %AEE&M'R..0Dt$ % s. %AEE&M'R..0Dt$ %   T "DH "S$bmm< "A!r((*Aq! "" CUD(C((  d?   7 Js '&' 4    sHHHAH* Red Wing navigation described in docs, then ensure /footwear-rwbr.z%//a[contains(@href,'/footwear-rwbr')]z0//button[contains(@data-target,'footwear-rwbr')]zx//*[self::a or self::button][contains(normalize-space(.), 'Red Wing') and not(contains(normalize-space(.), 'Heritage'))]Frr&rAr'Trcptt|jjxsdj vS)Nr#)FOOTWEAR_RWB_SLUGr rr^rrs rJz-open_red_wing_footwear_page..s)'Xamm-D-I-I-OR,V,V,XYrLN) rrrr)r*r+rmrrrqrrxrr)rhr,clickedrrr-s rJopen_red_wing_footwear_pager5js^ :; EF N OI G C vq)//0J0JBPS90UVB  ! !"ceg h JJt  C G  " ,-fb!'' Y  C%%&=rB C         V0 1   ssAC>C)C>/D DC;8C>:C;;C>> D  D  DD E &D88 EE EE c |jxst}g}gd}|D]_} |jtj |D]9}|j dxsdj}|s)|j|;a |jtj dD]B}dD];}|j |xsdj}d|vs+|j|=D |jxsd} | rtj| } tjd| tjD]"} |j| jd $tjd | tjD]"} |j| jd $g} t!} |D]}t#||} t%|}|j&xsdj)}|j*xsdj)}t,|vr`tj.d |sw|j0d |j&|}|| vs| j3|| j|t| vr| j5d t| S#t$r t}YwxYw#t$rYwxYw#t$rYwxYw#t$rd} YwxYw#t$rY1wxYw) zGDiscover footwear brand catalog paths from RW site navigation/elements.)za[href*='/footwear-']z[id*='footwear' i] a[href]znav a[href*='footwear']rr#z&[data-target], [data-url], [data-href])z data-targetzdata-urlz data-hrefrz/footwear-[a-z0-9\-]+rrz)https?://[^\s\"'<>]*/footwear-[a-z0-9\-]+z/footwear-[a-z0-9\-]+$z://)rrrxrrrrrTrrrrrRrrrrr r netlocrr^ SITE_HOSTrschemerinsert)rhrr dom_selectorsrr-rattrrIrr rurlsrrrrhostr^norms rJdiscover_footwear_catalog_urlsr@s'%%<)<JM  **2??C@ ,((06B==?%%d+ , &&r8`a )B@ )%%d+1r88:1$%%a( ) )!!'R   &5t2==Q *A   aggaj ) *I4WYWdWde *A   aggaj ) *DUD Hc "  AHHN))+DFFLb'')D D yy2D9 ((3qxxj/ t  HHTN KK $$& A*+ KC '&'"       "   sgJAJ&+J&AJ6J6(KAKJ#"J#& J32J36 KK KK K%$K%cBdd}t||j|y)zVWait for shared footwear-page elements instead of requiring product links immediately.c |jtjd}|ry |j tj dj xsdj}d|v}d|v}d|v}d |v}d |v}d |vxs d |jxsdjv}|xr|xs|xs |xs|xs|S#t$rYwxYw#t$rYywxYw) uCheck that the catalog UI has loaded. On the current Red Wing site, the grid of products is present in ``li.js-product-grid-item`` elements. Earlier heuristics looked for text such as "Sort By", "Results" or "Filters"; those still apply but may not appear until the user scrolls. We consider the catalog ready when either at least one product tile exists or legacy text heuristics are satisfied. zli.js-product-grid-itemTr{r#Fresultzsort byfiltersz more resultsrfootwearr) rrrrxrrr_rr) rtilestxt has_resultshas_sort has_filtershas_more has_searchhas_footwear_ctxs rJ_readyz/wait_for_footwear_catalog_ready.._readys OOBOO5NOE >>"++v6;;ArHHJC#o #3& !S(_ %,] ATRT@[@[@]0]fX%e%e %ex%e[ef     s""B3B? C CNrr)rr))rhrrNs rJwait_for_footwear_catalog_readyrPsg:&'"((0rLct|d}t|}d}d}|jddxs|jd}|ru|jd}|jd }|r<|jd d }t j d |} | r| j d}|r|jd d }|s|jdD]} tj| jd } t| tr| n| g} | D]} t| ts|s'| jdxsdj}|r|}|s^dD]Y}t!| j|xsdj}t j d|}|sH|j d}n|s|sn|s|sn|s}dD]x} |j#|}|s|j%dr$|jdxsdj}n|jd d }|scd|j'vsv|}n|s |rd|n|xsd}|s|j}|s|}|||fS#t$rYwxYw#t$rd}YwxYw)zMExtract (style_number, name, style_text) from legacy and RW order-site pages.rr#r shoeguide)class_ printSpacing)idrstrongrPT)rTz #\s*(\d+)rz"script[type='application/ld+json']r)rmpn productID \b(\d{3,6})\b)rrrzmeta[property='og:title']Nmetacontentzred wing for businessr()rextract_style_from_urlfindget_textrRrrrrsrtrx isinstancelistdictrwrTr$ select_onerr)rrr style_numberr style_text guide_divh3_tag strong_tagmatchscriptpayloadobjsobjnmrrrrr-rGs rJ extract_style_and_name_from_htmlrns } -D)#.L DJ % 4T ^ 8TI%^^H- D9JIIlJ7E${{1~ &&s$&7D kk"FG F **V__4_%@A)$77gYD !#t,''&//R668B!#:"!#''#,"4"5;;=II&6<+,771:L! " L   / 2 E C __S)~~f%vvi(.B557kk#Tk2.ciikA  1=w|n-DJB !'')  z ))Y  4  s$%I&I6& I32I36 JJcgd}tjdz}tj|krd}|D]W} |jtj|}|j r$|j r|j d}nY|sytjdtj|kryy#t$rYwxYw)N)zN//button[contains(.,'Agree') or contains(.,'Accept') or contains(.,'Proceed')]zI//a[contains(.,'Agree') or contains(.,'Accept') or contains(.,'Proceed')]uR//button[contains(.,'Close') or contains(.,'×') or contains(@aria-label,'Close')]g@FT皙?) rrrrrrrrrxrq)rhxpathsr"r4xpr-s rJrrAsF ))+ C ))+  B ((26??$HHJ"G    3 ))+   sAC CCc d}d}t|D]Q}t|d}dD]} |jtj|}|j s5|j d}t|dr|jsc|dvrh |jd|tjd  |jd }tjd nt|} t| D chc]%} | j d s| j d 'c} } | |k(r|dz }nd}| }|s|dk\ry |jdtjdTy#t$rYwxYw#t$r( |jd |n#t$rYY|wxYwYwxYw#t$rYwxYwcc} w#t$rY|wxYw)NrF)zz//button[contains(translate(normalize-space(.),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'more results')]zu//a[contains(translate(normalize-space(.),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'more results')]disabledr)Nfalser#r&rAr'T333333?rrr&z/window.scrollTo(0, document.body.scrollHeight);g?)r|rrrrrrr~rrmrxrrrqrrr) rh max_rounds stable_rounds last_count_ clicked_morerrr! disabled_attrlinksrcounts rJscroll_to_load_allrXsMJ : ?v  % B! ))"((B7'') # 1 1* = 3 -cnn6F (;;))Y  4 !IIK $  3G% N&f-eWqv?VQ__V,WX J  Q MMJ  2    ! !"S T 3?8! !!!--.EsK$! !L!  X   s0F)-F)F) E&F)4E5F)2F9 F9 <F>& E2/F)1E22F)5 F&?FF& F F&F)F F&#F)%F&&F)) F65F6> G  G c|xsdj}tjdd|}|sy|j}|dvrytjd|ryd|vryt |dkry|S)Nr#rOrP> QUICK VIEW ADD TO CART VIEW DETAILSrrrrr'z\d{2,}$r&)rTrRrSrrr)rUrs rJ_clean_listing_namersu bA vsAA   A   ||Iq! ax 1vz HrLcxg} |j|jdxsd |j|jdxsd |j|jxsdg}|D]B}|st |j D]!}t |}|s|j|#Dt}g}|D])}||vs|j||j|+|sydd}|j|d|dS#t$rYwxYw#t$rYwxYw#t$rYwxYw) zJBest-effort: read the product name as displayed on a catalog listing tile. aria-labelr#rc*t}tjdr|dz }tjdjr|dz }tjdjr|dz }t fddDr|d z}|S) Nz[A-Za-z] \bWOMEN'?S\brD \bMEN'?S\bc3BK|]}|jvywr)r.0rHrUs rJ z;_listing_name_from_anchor..score..sX!qAGGI~Xs)viewquickcartcomparewishlistr@)rrRrrany)rUscs` rJscorez(_listing_name_from_anchor..scoresz V 99[! $ "HB 99_aggi 0 "HB 99]AGGI . !GB X'WX X "HB rLT)rreverser)rUr$rrv) rrrxr_r$ splitlinesrrrsort)rrlinesclnruniqrs rJ_listing_name_from_anchorrsRJ !//,7=2> !//'28b9 !&&,B' E ! a&##% !B$R(B R  !! 5D D T> HHRL KKO   II%I& 7NY            s4$D$DD- DD D*)D*- D98D9c|r|syd}|D]l}t|trt|dkr"|dxsdj}|s discovered z candidate product links zcatalog=international)rr$rr.r@r`rqrstderrrrrvrrPrrrrrrrr\"PREFER_INTERNATIONAL_LISTING_NAMESr) rh all_links seen_href seen_stylerhrdrr page_linksrrrs rJcollect_product_linksrs I%I5J&(OV01&!1&9OT,=OLJJ?L@Q?RRTUV 6 .qc456)#%';v6"!# &v. $A??6*D#D)   d #  $.hs 'a16 'r$F)rTrrr)rU junk_tokenss` rJ_looks_like_junkrbsI b!A WK '; '' 1v{ rLc |dxsdj}|dxsdj}|sy|jtk(ryy#t$rYywxYw)Nrr#rTF)rTrxrBAD_NAME_SENTINEL)rowrrs rJ _is_bad_rowrpsdQ2$$&A "##%  zz|((  s.A AAc|jdry|jdgxsg}t|jdgxsg}d}t}|D]}t|tr|s|dxsdj }|s0t |t kr|j|Tt |dkDr|dxsdj nd}t |dkDr|dxsdj nd}|r|r|j||syt|D]+} t| } | s| |vs|j| |dz }-t||d<|S) zIf checkpoint rows are missing URL/Image columns or have empty URL/Image, requeue those links. Runs at most once per checkpoint unless you delete/clear ck['media_repair_done']. rirrgrfr#rCr&r) rwrr_r`rTr EXPECTED_COLSrr\discardsorted) rLrgrfremoved styles_neededr&rurl_cellimg_cellrrIs rJrepair_missing_mediarsZ vv!" 66&"  #DRVVL"-34JGEM  %!T"! 1""$  q6M !   e $ +.q6A:AaDJB%%'2+.q6A:AaDJB%%'2x   e $ % Z  #D ) " %   t $ qLG  j)B| NrLc t|jdgxsg}t|jdgxsg}|r|sy|Dchc]}|st|s|d}}|syd}t|D]+}t |}|s||vs|j ||dz }-|rt ||d<t|jdixsi}t|jD]'} t | }|s||vs|j| d)||d<|jdgxsgD cgc]} t | |vr| c} |d<t||Scc}wcc} w#t$rYywxYw)aaIf checkpoint contains obviously bad rows, un-mark those links as 'done'. This fixes the situation where a previous run captured the site header/ads into the Name/Brand columns and those rows are now 'stuck' because resume logic skips already-done links. Returns: Number of links that were re-queued (removed from done_links). rgrfrrrjNrk) r`rwrrr\removerrakeyspoprrx) rLrgrfr& bad_stylesrrrIfckrs rJrepair_bad_checkpoint_rowsrs$BFF62&,"- b17R8 :$(AqA+a.adA A$ D'-BbJ&!!$'1    %j1B| bff]B/526B"'')_ $+A." *FF1dO $!#B} 4666:Mr3R3XVX'VQ)?)B*)T()'VB" # B 9B.'V saAE&E& E EE$E&)E&E& A0E&=E&1E&3E! E& E&& E21E2c |xsdjDcgc]}|j}}t|D]\}}|s |j|s|t |djd}|r|cSt |dzt |dzt |D]!}||xsdj}|s|ccSyycc}w)u Parse a simple field/value from extracted page text. Red Wing pages often render fields in tables so the extracted text looks like: Name DynaForce® We support both 'Name: DynaForce' and 'Name' on one line with the value on the next. r#Nz : rr)rrT enumeraterrr|r) text_block field_namerrirtailjnxts rJparse_field_liners$.#3"?"?"A BBRXXZ BE BU# 4  ??: &J()//7D 1q5#a!eSZ"89 Qx~2,,.J    CsCcN|xsdj}tjd|tj}|rz|j dj}|rYg}|j D]3}|j |jr|n|j5dj|S|jD]}tjd|tjs*tjd|jtj}|r||j dj}|r[g}|j D]3}|j |jr|n|j5dj|cSyy)Nr#z^(.*?)\s+style\s*#\s*\d+rrrPz\bstyle\s*#\s*\d+\bz^(.*?)\s+style\s*#\s*\d+\b) rTrRrrrsplitrisupper capitalizerr) header_text body_texthtrrr rrm2s rJ extract_brandrsE   " " $B -rGAggaj  CYY[ A  1@ A88C= $$&  99+T G8$**,bmm\Bhhqk'')C YY[I  1HI88C=(    rLc|rM|jjd|}|dk7r&|||dz}t|d}|r|jSt|d}|r|jSdS)Nz ABOUT THE rti@r)r#)rr]rrT)rridxchunkvals rJextract_about_namer st oo$$z%%9: "9c3:.E"5&1Cyy{" 9f -C399;%2%rLc "tdtjd|jD}|tdtjd|jDz}t ddDcic] }|d||v c}Scc}w)Nc32K|]}t|ywrrvrrs rJrz parse_heights..sV1AVz\b(\d{1,2})\s*-\s*INCH\bc32K|]}t|ywrrrs rJrz parse_heights..sSAQSrz\b(\d{1,2})\s+INCH\b ")rrRfindallrr|)r_foundrs rJ parse_heightsrst V +F UV VE SS,CTZZ\!RS SSE+0B< 8aqcGa5j 88 8s:B c|xsdj}|jd}|jd}|jd}d|vr|jdsd|vrd}|||fS)Nr#zred wingz irish setterworxz by red wingT)rr) brand_strbis_rwis_isis_worxs rJclassify_brand_familyrsi b!A LL $E LL (Ell6"G! all62!9K %  rLc|xsdj}tjdd|}|jdd}tjdd|tj}|jjdd}tjdd|}|sd }t ||kDr|d|j d}|S) Nr#rOrP®z [^\w\-\.\s]+rr{z_+boot)rTrRrSrQUNICODErr)rmax_lenrUs rJ_safe_filename_from_namer-s A vsAA $A ARZZ8A  #s#A uc1A  1v hwK  s # HrLctjjdz j}|j dd|S)NImagesTr)rcwdparentresolverr2s rJ _images_dirr;s6   X %..0AGGD4G( HrLc6ttddz |z S)Nz..r)r$r)fnames rJ_rel_image_pathrAs tDzH$u, --rLc dD]9} |jtj|}|r|jr|cS; |jtj d}|jtjd}|r|jr|S |j tjd}|s! |j tjd}d}d}|D].} |js|jxsi}t|jdxsd} t|jdxsd} | d ks| d krm|jd xsd j |jd xsd j |jd xsd j| | z} t fddDr| dz} t fddDr| dz} tfddDr| dz} | |kDr| }|}1|S#t$rYwxYw#t$rYwxYw#t$rg}YwxYw#t$rg}YzwxYw#t$rYwxYw)N)@div.slick-slide.slick-current.slick-active img[itemprop='image'].div.slick-slide.slick-current.slick-active imgHdiv.c-image-carousel__slider-item.js-carousel-item img[itemprop='image'] productImageimgmain imgrrrheightxaltr#classsrcc3&K|]}|v ywrrH)rrrs rJrz1_pick_best_product_img_element..vsP18Pr)rshoechukkahikermocrwc3&K|]}|v ywrrH)rrclss rJrz1_pick_best_product_img_element..xsM18Mr)productprimaryheroimagegffffff?c3&K|]}|v ywrrH)rrrs rJrz1_pick_best_product_img_element..zsR18Rr)z /dw/imagescene7z/imagesstaticg?)rrrrrxr(rrsizerrwrrr)rhrr  containerimgsbest best_scorer-szrhrrrrs @@@rJ_pick_best_product_img_elementr)Ese   %%boos;Cs'')   ''~> $$R__e< 3##%J ##BOOZ@  '' U;D DJ ??$BBbffWo*+AbffX&+!,A3w!c'##E*0b779C##G,299;C##E*0b779CEEP%OPP M%LMM R%QRR z!" +0 Kc         D 6   sh3HAH% H5; I&I7AICI H"!H"% H21H25 II II I&%I&cdD]u} |jtj|}|jdxsdj }|jdxsdj }|s|r|xs|cSw |jtj d} |jtjd}|rP|jdxsdj }|jdxsdj }|s|r|xs|SdD]u} |jtj|}|jdxsdj }|jdxsdj }|s|r|xs|cSwy#t $rYwxYw#t $rd}YwxYw#t $rYwxYw#t $rYwxYw)N)r r r z6div.c-image-carousel__slider-item.js-carousel-item imgzoomimgr#rr r )z$li[data-orbit-slide="product-1"] imgz li.active imgzul#productImage li.active imgzmain li.active imgr)rrrrrTrxr()rhrr rFrUr#s rJ_extract_product_image_urlr,s   %%boos;C""9-3::  ((%@C ""9-3::.png). Preferred: download the product image URL (zoomimg/src) for best quality. Fallback: element screenshot if download fails. Returns relative path like ../Images/__595.png (or empty string). r#__.pngz&&rz Mozilla/5.0r@)rrNr&rp)rTrr,rQrequestsrwokr[ write_bytesrrxr)rmrrrqr screenshotr$save_screenshot)rhrrout_pathimg_urlrr&img_els rJcapture_product_image_tempr9s[b   !E t E}u$H  ,V4 oogs3G#]3G WgrBAtt $$QYY/&u-- F/7  _agh 3 v#h-(u%%5               " "3x= 1"5) )   sxA8D( D7'D0 D?+$E DD D-,D-0 D<;D<? E  E  F$E?<F? F F F  Fc|sytj|z j}|js|St |}|d}t |z }|jr`|d|d}t |z }|jr;t ddD],}|d|d|d}t |z }|jr,n |j|t|S#t$r|cYSwxYw)zNRename ../Images/__