Question

这是我的剧本：

url = "https://www.walmart.com.mx/productos?Ntt=notebook"
agent = {"User-Agent":'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}
page = requests.get(url, headers=agent)
soup=BeautifulSoup(page.content,'lxml')
print(soup)
urls = []
for p_tag in soup.findAll('p'):
    urls.append = p_tag
print(urls);

网址列表返回null。

打印请求将返回此文件，但是在此文件中，我检查了它是否等于浏览器控制台上的实际文件。谁能帮助我成为该主题的初学者。

<!DOCTYPE html>
<html lang="en"><head><meta charset="utf-8"/><meta content="IE=edge" http-equiv="x-ua-compatible"/><meta content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no,shrink-to-fit=no" name="viewport"/><meta content="#000000" name="theme-color"/><script type="text/javascript">(function(){(function(){function a(){if(void 0===b.dialogArguments)return navigator.cookieEnabled;document.cookie="__dTCookie=1";var a=-1!==document.cookie.indexOf("__dTCookie");document.cookie="__dTCookie=1; expires=Thu, 01-Jan-1970 00:00:01 GMT";return a}if(window.dT_)window.console&&window.console.log("Duplicate agent injection detected, turning off redundant initConfig.");else{var b="undefined"!==typeof window?window:self;a()&&(window.dT_||(window.dT_={cfg:"domain=walmart.com.mx|reportUrl=/rb_3770e195-dd69-4828-8cae-1067062e79aa|app=8f56071731344bd0|ssc=1|featureHash=A27SVfjqrux|srsr=25000|rdnt=1|uxrgce=1|bp=2|srms=1,1,,,|uxrgcm=100,25,300,3;100,25,300,3|dpvc=1|lastModification=1572550684266|dtVersion=10175190917092722|tp=500,50,0,1|uxdcw=1500|agentUri=/ruxitagentjs_A27SVfjqrux_10175190917092722.js|auto=1|domain=walmart.com.mx|rid=RID_-233466361|rpid=-741352723|app=8f56071731344bd0",iCE:a}))}})();}).call(this);(function(){(function(){function Jb(){return ta?new ta:ua?new ua("MSXML2.XMLHTTP.3.0"):d.XMLHttpRequest?new d.XMLHttpRequest:new d.ActiveXObject("MSXML2.XMLHTTP.3.0")}function Kb(){ua=ta=void 0}function t(){var a=0;try{a=d.performance.timing.navigationStart+Math.floor(d.performance.now())}catch(b){}return 0>=a||isNaN(a)||!isFinite(a)?(new Date).getTime():a}function Y(a,b){function c(){delete la[g];a.apply(this,arguments)}for(var e=[],I=2;I<arguments.length;I++)e[I-2]=arguments[I];var g;"apply"in va?g=va.apply(d,[c,b].concat(e)):g=va(c,b);la[g]=!0;return g}function Za(a){delete la[a];"apply"in Z?Z.call(d,a):Z(a)}function k(a){for(var b=[],c=1;c<arguments.length;c++)b[c-1]=arguments[c];a.push.apply(a,b)}function $a(a){k(aa,a)}function Lb(a){for(var b=aa.length;b--;)if(aa[b]===a){aa.splice(b,1);break}}function Mb(){return aa}function Nb(a,b){return ab(a,b)}function Ob(a,b){a=new Pb([a],{type:b});return Qb(a)}function Rb(a,b){return bb?new bb(a,b):void 0}function Sb(a){"function"===typeof a&&k(cb,a)}function Tb(){return cb}
function Ub(){return Da}function db(a){return function(){for(var b=[],c=0;c<arguments.length;c++)b[c]=arguments[c];if("number"!==typeof b[0]||!la[b[0]])try{return a.apply(this,b)}catch(e){return a(b[0])}}}function Vb(){ma&&(d.clearTimeout=Z,d.clearInterval=Ea,ma=!1)}function ba(a,b){return isNaN(a)||isNaN(b)?0:Math.floor(Math.random()*(b-a+1))+a}function u(a,b){return parseInt(a,b||10)}function p(a,b,c){void 0===c&&(c=0);var e=-1;b&&a&&a.indexOf&&(e=a.indexOf(b,c));return e}function eb(a){return document.getElementsByTagName(a)}
function fb(a){var b=a.length;if("number"===typeof b)a=b;else{for(var b=0,c=2048;a[c-1];)b=c,c+=c;for(var e=7;1<c-b;)e=(c+b)/2,a[e-1]?b=e:c=e;a=a[e]?c:b}return a}function Wb(a){a=encodeURIComponent(a);var b=[];if(a)for(var c=0;c<a.length;c++){var e=a.charAt(c);k(b,Xb[e]||e)}return b.join("")}function O(a){if(!a)return"";var b=d.crypto||d.msCrypto;if(b){var c=new Int8Array(a);b.getRandomValues(c)}else for(c=[],b=0;b<a;b++)c.push(ba(0,32));a=[];for(b=0;b<c.length;b++){var e=Math.abs(c[b]%32);a.push(String.fromCharCode(e+
(9>=e?48:55)))}return a.join("")}function gb(){return!(!d.console||!d.console.log)}function Yb(){try{Zb.apply(d.parent,arguments)}catch(a){}}function $b(){try{ac.apply(d.top,arguments)}catch(a){}}function bc(a){var b=Array.prototype.slice.call(arguments,1);try{cc.apply(a,b)}catch(c){}}function dc(a){var b=Array.prototype.slice.call(arguments,1);try{ec.apply(a,b)}catch(c){}}function z(){return d.dT_}function fc(){return A}function gc(){return hb}function hc(){return ib}function ic(){return wa}function jb(){return"dtAdk"}
function jc(){return ca}function kb(a){-1<d.dT_.io(a,"^")&&(a=a.split("^^").join("^"),a=a.split("^dq").join('"'),a=a.split("^rb").join(">"),a=a.split("^lb").join("<"),a=a.split("^p").join("|"),a=a.split("^e").join("="),a=a.split("^s").join(";"),a=a.split("^c").join(","),a=a.split("^bs").join("\\"));return a}function kc(){return P}function lc(a){P=a}function lb(a){var b=d.dT_,c=b.scv("rid"),b=b.scv("rpid");c&&(a.rid=c);b&&(a.rpid=b)}function mb(a){if(a.xb){a=kb(a.xb);try{P=new RegExp(a)}catch(b){}}}
function nb(a){var b={};a=a.split("|");for(var c=0;c<a.length;c++){var e=a[c].split("=");2===e.length&&(b[e[0]]=decodeURIComponent(e[1].replace(/\+/g," ")))}return b}function Fa(){var a=l("csu");return(a.indexOf("dbg")===a.length-3?a.substr(0,a.length-3):a)+"_"+l("app")+"_Store"}function xa(a,b,c){b=b||{};a=a.split("|");for(var e=0;e<a.length;e++){var d=a[e],g=p(a[e],"=");-1===g?b[d]="1":(d=a[e].substring(0,g),b[d]=a[e].substring(g+1,a[e].length))}!c&&(c=b,a=c.spc)&&(e=document.createElement("textarea"),e.innerHTML=a,c.spc=e.value);return b}function Q(a){return a in f?f[a]:ya[a]}function m(a){a=Q(a);return"false"===a||"0"===a?!1:!!a}function J(a){var b=u(Q(a));isNaN(b)&&(b=ya[a]);return b}function l(a){return String(Q(a)||"")}function mc(a,b){f[a]=b}function ob(a){return f=a}function pb(a){var b=location.hostname;return b&&a?b===a||-1!==b.indexOf("."+a,b.length-("."+a).length):!0}function Ga(a){f[a]=0>p(f[a],"#"+a.toUpperCase())?f[a]:""}function Ha(a){var b=a.agentUri;b&&-1<p(b,"_")&&(b=/([a-zA-Z]*)[0-9]{0,4}_([a-zA-Z_0-9]*)_[0-9]+/g.exec(b))&&b.length&&2<b.length&&(a.csu=b[1],a.featureHash=b[2])}function Ia(a,b){pb(f.domain||"")||(f.domainOverride=location.hostname+","+f.domain,delete f.domain);f.pVO&&(a.pVO=f.pVO);b||(a.bp=a.bp||ya.bp,1===h&&a.bp1&&(a.bp=1),a.bp2&&(a.bp=2),4!==a.bp||d.JSON||(a.bp=1))}function nc(){return f}function da(a,b){try{var c=na;c&&c.setItem(a,b)}catch(e){}}function ea(a){try{var b=na;if(b)return b.getItem(a)}catch(c){}return null}function R(a){try{var b=na;b&&b.removeItem(a)}catch(c){}}function za(a,b){if(S()&&(!z().A||qb))return a.apply(this,b||[])}function S(){return!m("coo")||m("cooO")||qb}function n(a){document.cookie=a+'="";path=/'+(l("domain")?";domain="+l("domain"):"")+"; expires=Thu, 01 Jan 1970 00:00:01 GMT;"}function rb(a,b,c){var e=1,d=0;do document.cookie=a+'=""'+(b?";domain="+b:"")+";path="+c.substr(0,e)+"; expires=Thu, 01 Jan 1970 00:00:01 GMT;",e=c.indexOf("/",e),d++;while(-1!==e&&5>d)}function K(a){var b=document.cookie;if(!b)return"";var c=a+"=";a=p(b,c);if(0>a)return"";for(;0<=a;)if(a&&" "!==b.charAt(a-1)&&";"!==b.charAt(a-1))a=p(b,c,a+c.length);else return c=a+c.length,a=p(b,";",a),0<=a?b.substring(c,a):b.substr(c);return""}function oc(a,b,c,e){b||0===b?(b=(""+b).replace(/[;\n\r]/g,"_"),a=a+"="+b+";path=/"+(l("domain")?";domain="+l("domain"):""),c&&(a+=";expires="+c.toUTCString()),e&&(a+=";Secure"),document.cookie=a):n(a)}function B(a,b,c,e){za(oc,[a,b,c,e])}function oa(a){var b=/^[0-9A-Za-z_=:\$\+\/\.\-\*%\|]*$/.test(a);return a&&2<a.split("$").length?!1:b}function sb(){var a=K(A);a||((a=ea(A))&&oa(a)?T(a):a="");return oa(a)?a:""}function T(a){B(A,a,void 0,m("ssc"))}function pa(a){return 32===a.length||12>=a.length?a:""}function tb(a){if(!isNaN(Number(a))){var b=u(a);if(-99<=b&&99>=b)return a}return""}function ub(a){var b={sessionId:"",b:""},c=p(a,"|"),e=a;-1!==c&&(e=a.substring(0,c));c=p(e,"$");-1!==c?(b.sessionId=pa(e.substring(c+1)),b.b=tb(e.substring(0,c))):b.sessionId=pa(e);return b}function vb(a){var b={sessionId:"",b:""};a=a.split("v"===a.charAt(0)?"_":"=");if(2<a.length&&!(a.length%2)){var c=Number(a[1]);if(isNaN(c)||3>c)return b;for(var c={},e=2;e<a.length;e++)c[a[e]]=a[e+1],e++;c.sn?b.sessionId=pa(c.sn):b.sessionId="hybrid";c.srv&&(b.b=tb(c.srv));"1"===c.ol&&(da("dtDisabled","true"),z().disabled=!0,z().A=!0)}return b}function C(a){var b=document.cookie?document.cookie.split(a+"=").length-1:0;if(1<b){var c=l("domain")||d.location.hostname,e=d.location.hostname,I=d.location.pathname,g=0,f=0;D.push(a);do{var h=e.substr(g);if(h!==c||"/"!==I){rb(a,h===c?"":h,I);var k=document.cookie?document.cookie.split(a+"=").length-1:0;k<b&&(D.push(h),b=k)}g=e.indexOf(".",g)+1;f++}while(g&&10>f&&1<b);l("domain")&&1<b&&rb(a,"",I)}}function pc(){C(ca);C(A);C(wa);C("rxvt");$a(function(a,b,c,e){0<D.length&&!b&&(a.av(e,0,"dCN",function(){return D.join(",")}),a.av(e,4,"duplicateCookieNames",function(){return D.slice()}),D=[])})}function qc(){return fa}function Aa(a,b,c,e,d){var g=document.createElement("script");g.setAttribute("src",a);b&&g.setAttribute("defer","true");c&&(g.onload=c);e&&(g.onerror=e);d&&g.setAttribute("id",d);g.setAttribute("crossorigin","anonymous");a=document.getElementsByTagName("script")[0];a.parentElement.insertBefore(g,a)}function Ja(a,b){return Ka+"/"+(b||U)+"_"+a+"_"+(J("buildNumber")||z().version)+".js"}function Ba(a){S()?a():(v||(v=[]),k(v,a))}function rc(a){return za(a)}function sc(){for(var a=0;a<v.length;a++)Y(v[a],0);v=[];f.cooO=!0}function tc(){f.cooO=!1;n(A);n(ca);n(wa);n("dtSa");n(jb());0===h&&(n("rxVisitor"),n("rxvt"));try{var a=na;a&&(0===h&&a.removeItem("rxVisitor"),a.removeItem(A));(a=La)&&a.removeItem(Fa())}catch(b){}}function wb(a){if(a=a||sb()){var b=a.charAt(0);return"v"===b||"="===b?vb(a):ub(a)}return{sessionId:"",b:""}}function V(a){return wb(a).b}function ga(a){return wb(a).sessionId}function uc(){return q}function vc(){Ba(function(){ga()||T((0===h?-1*ba(2,21)+"$":"")+O(32));q=V()||""})}function L(a,b){try{d.localStorage&&d.localStorage.setItem(a,b)}catch(c){}}function qa(a){try{if(d.localStorage)return d.localStorage.getItem(a)}catch(b){}return null}
function ha(a){try{d.localStorage&&d.localStorage.removeItem(a)}catch(b){}}function xb(a,b){b=E(b);for(var c=!1,e=0;e<b.length;e++)b[e].frameId===fa&&(b[e].g=a,c=!0);c||k(b,{frameId:fa,g:a});ia(b)}function ia(a,b,c){if(a){var e=0===h;var d=[];for(var g=0;g<a.length;g++)if("-"!==a[g].g){0<g&&0<d.length&&k(d,"p");var f=q;f&&(k(d,f),k(d,"$"));k(d,a[g].frameId);k(d,"h");k(d,a[g].g)}e&&!d.length&&(Ma&&(M(0,!0,"a"),Na(!1)),q=V()||"",k(d,q),k(d,"$"),k(d,fa),k(d,"h-"));a=e?b||Oa():W();if(e||a)k(d,"v"),k(d,a),e="undefined"!==typeof c?c:r(),0<=e&&(k(d,"e"),k(d,e));d=d.join("")}else d="";d||0!==h||(Ma&&(M(0,!0,"a"),Na(!1)),q=V()||"",c="undefined"!==typeof c?c:r(),d=q+"$"+fa+"h-v"+(b||Oa()+(0<=c?"e"+c:"")));B(ca,d||"-",void 0,m("ssc"))}function E(a){var b=K(ca),c=[];if(b&&"-"!==b){for(var b=b.split("p"),d="",f=null,g=0;g<b.length;g++){var h=b[g],m=p(h,"h"),l=p(h,"v"),n=p(h,"e"),w=h.substring(p(h,"$")+1,m),m=-1!==l?h.substring(m+1,l):h.substring(m+1),d=d||-1!==l?-1!==n?h.substring(l+1,n):h.substring(l+
1):"",f=f||-1!==n?h.substring(n+1):null;(h=a)||(h=u(w.split("_")[0]),l=t()%Pa,l<h&&(l+=Pa),h=h+9E5>l);h&&k(c,{frameId:w,g:"-"===m?"-":u(m)})}for(g=0;g<c.length;g++)c[g].visitId=d||"",c[g].j=null!==f?u(f):-1}return c}function Oa(){return W()||M(0,!0,"c")}function W(){var a=E(!0);if(!(F()<=t())){X(!1);if(1<=a.length)return-1!==r()&&2<=J(Qa)&&a[0].j>=J(wc)?M(0,!0,"e"+a[0].j):a[0].visitId||"";(a=qa(ja))||(a=ea(ja));return a||""}return""}function X(a){var b=t(),c=yb().m;a&&(c=b);zb(b+Ab+"|"+c);Bb()}function Cb(a,b){a||(a=ba(1,1E6));var c=ga()||"";c||(c=(0===h?-1*ba(2,21)+"$":"")+O(32),T(c),c=ga(c)||"");a=""+a;for(var d=a.length,f=[],g=0;g<c.length;g++)f[g]=String.fromCharCode(65+Math.abs((c.charCodeAt(g)^a.charCodeAt(g%d))%26));c=f.join("");for(a=0;a<Ra.length;a++)Ra[a](c,N,b);return c}function Sa(a){var b=E(!1),c=2<=J(Qa)?0:-1;ia(b,a,c);Ta(ja,a);L(Ua,String(c));X(!0)}function M(a,b,c){b&&(N=!0);a=Cb(t(),c);Sa(a);return a}function xc(a){Ra.push(a)}function Bb(){Va&&Za(Va);Va=Y(Db,F()-t())}function Db(){if(F()<=t()&&S()){var a=t(),a=Cb(a,"t"+(a-F()));Sa(a);return!0}Ba(Bb);return!1}function zb(a){B("rxvt",a,void 0,m("ssc"));Ta("rxvt",a)}function Ta(a,b){m("dpvc")||m("pVO")?(da(a,b),ha(a)):(L(a,b),R(a))}function Eb(){var a=K("rxvt");a||(a=qa("rxvt")||"")||(a=ea("rxvt")||"");return a}function Fb(){var a=W()||"";Ta(ja,a);a=Eb();zb(a)}function yb(){var a={v:0,m:0},b=Eb();if(b)try{var c=b.split("|");2===c.length&&(a.v=parseInt(c[0],10),a.m=parseInt(c[1],10))}catch(e){}return a}function F(){var a=yb();return Math.min(a.v,a.m+Gb)}function yc(a){Ab=a}function Na(a){void 0===a&&(a=!0);Ma=a}function zc(){var a=N;N=!1;return a}function Ac(){Db()||X(!1)}function Bc(){if(0===h&&-1!==r()&&2<=J(Qa)){var a=E(!1),b=r()+1;ia(a,"",b);L(Ua,String(b))}}function r(){var a=E(!0);if(1<=a.length&&!isNaN(a[0].j))return a[0].j;a=qa(Ua)||"";a=u(a);return isNaN(a)?-1:a}function Wa(){var a=K("rxVisitor");if(!a||a.length&&a.length!==Xa)a=qa("rxVisitor")||ea("rxVisitor"),a&&a.length===Xa||(Hb=!0,a=t()+"",a+=O(Xa-a.length));var b=a;if(m("dpvc")||m("pVO"))da("rxVisitor",b);else{var c=new Date;c.setFullYear(c.getFullYear()+2);L("rxVisitor",b)}B("rxVisitor",b,c,m("ssc"));return a}function Cc(){return Hb}function Dc(a){var b=K("rxVisitor");n("rxVisitor");R("rxVisitor");ha("rxVisitor");B("rxVisitor",b);f.pVO=!0;a&&L(Ya,"1");Fb()}function Ec(){ha(Ya);m("pVO")&&(f.pVO=!1,Wa());Fb()}function Fc(){var a=d.dT_;d.dT_={version:"10175190917092722",cfg:a?a.cfg:"",iCE:a?a.iCE:function(){return navigator.cookieEnabled},ica:1,disabled:!1,A:!1,gx:Jb,cx:Kb,mp:Yb,mtp:$b,mi:bc,mw:dc,gAST:Ub,ww:Rb,stu:Ob,nw:t,apush:k,st:Y,si:Nb,aBPSL:$a,rBPSL:Lb,gBPSL:Mb,aBPSCC:Sb,gBPSCC:Tb,buildType:0===h?"dynatrace":"appmon",gSSV:ea,sSSV:da,rSSV:R,rvl:ha,pn:u,iVSC:oa,p3SC:vb,pLSC:ub,io:p,dC:n,sC:B,esc:Wb,gSId:V,gDtc:ga,gSC:sb,sSC:T,gC:K,cRN:ba,cRS:O,gEL:fb,gEBTN:eb,gSCN:fc,gPCHN:gc,gRHN:hc,gPCCN:jc,gLCN:ic,gMSIDCN:jb,cfgO:nc,pCfg:nb,pCSAA:xa,cFHFAU:Ha,sCD:Ia,bcv:m,ncv:J,scv:l,stcv:mc,rplC:ob,cLSCK:Fa,gFId:qc,gBAU:Ja,iS:Aa,eWE:Ba,oEIE:rc,oEIEWA:za,eA:sc,dA:tc,gcSId:uc,iNV:Cc,gVID:Wa,dPV:Dc,ePV:Ec,sVIdUP:Na,sVTT:yc,sVID:Sa,rVID:W,gVI:Oa,gNVId:M,gARnVF:zc,cAUV:Ac,uVT:X,aNVL:xc,gPC:E,cPC:xb,sPC:ia,clB:Vb,ct:Za,aRI:lb,iXB:mb,gXBR:kc,sXBR:lc,de:kb,cCL:gb,gEC:r,iEC:Bc}}var G=window;if(!G.dT_||!G.dT_.cfg||"string"!=typeof G.dT_.cfg||G.dT_.initialized)G.console&&G.console.log("Initconfig not found or agent already initialized! This is an injection issue.");else if(!(navigator.userAgent&&0<=navigator.userAgent.indexOf("RuxitSynthetic"))){var d="undefined"!==typeof window?window:self,ta,ua,aa,cb=[],Da,La,na,la={},Xb=new(function(){return function(){this["!"]="%21";this["~"]="%7E";this["*"]="%2A";this["("]="%28";this[")"]="%29";this["'"]="%27";this.$="%24";this[";"]="%3B";this[","]="%2C"}}()),va,ab,cc=d.postMessage,bb=d.Worker,Pb=d.Blob,Qb=d.URL&&d.URL.createObjectURL,ec=d.Worker&&d.Worker.prototype.postMessage,Zb=d.parent.postMessage,ac=d.top.postMessage,Z,Ea,ma=!1,h,ya,ca="dtPC",A="dtCookie",hb="x-dtpc",ib="x-dtreferer",wa="dtLatC",P,f={},qb=!!navigator.userAgent&&0<=navigator.userAgent.indexOf("RuxitSynthetic"),D=[],fa,Pa=6E8,Ib,Ka,U,Gc={childList:!0,subtree:!0,attributes:!0,attributeOldValue:!0},Hc=["_DT_RENDERING_"],v=[],q,wc="mel",Qa="vs",Ua="rxec",ja="rxvisitid",Va,Ab=18E5,Gb=216E5,N=!1,Ra=[],Ma=!1,Ya="dt-pVO",Xa=45,Hb=!1;if(!function(a){try{h=a;var b=d.dT_;ta=d.XMLHttpRequest;ua=d.ActiveXObject;va=d.setTimeout;ab=d.setInterval;ma||(Z=d.clearTimeout,Ea=d.clearInterval);if(!((b.iCE?b.iCE():navigator.cookieEnabled)&&("complete"!==document.readyState||d.performance&&d.performance.timing)))return!1;Fc();try{La=d.localStorage,na=d.sessionStorage}catch(Ca){}Da=t();aa=[];la={};ma||(d.clearTimeout=db(Z),d.clearInterval=db(Ea),ma=!0);fa=Da%Pa+"_"+u(ba(0,1E3)+"");ya={ade:"",aew:!0,agentLocation:"",agentname:"",agentUri:"",uana:"data-dtname,data-dtName",app:"",async:!1,auto:!1,bandwidth:"300",bp1:!1,bp2:!1,bp:0===h?1:2,bs:!1,buildNumber:0,coo:!1,cooO:!1,cors:!1,csu:"",cux:!1,dataDtConfig:"",debugName:"",dASXH:0!==h,disableCookieManager:!1,disableLogging:!1,dmo:!1,dpvc:!1,disableXhrFailures:!1,domain:"",domainOverride:"",doNotDetect:"",dsndb:!1,dsss:!1,euf:!1,evl:"",extblacklist:"",exteventsoff:!1,fa:!1,featureHash:"",ffi:!1,hvt:216E5,lastModification:0,imm:!1,initializedModules:"",ign:"",instr:"",lab:!1,legacy:!1,lmut:!0,lzwd:!1,lzwe:!1,md:"",name:"",mdn:5E3,mel:200,mepp:10,moa:30,mrt:3,mpl:0===h?1024:100,msl:3E4,mhl:4E3,ncw:!1,ntd:!1,oat:180,ote:!1,perfbv:1,prfSmpl:0,pui:!1,pVO:!1,rdnt:0,reportUrl:"dynaTraceMonitor",restoreTimeline:!1,rid:"",ridPath:"",rpid:"",rt:0===h?1E4:0,rtl:0===h?0:100,rtp:0===h?2:1,rtt:1E3,rtu:200,rx_visitID:"",sl:100,sosi:!1,spc:"",srbbv:1,srbw:!0,srad:!0,srmr:100,srms:"1,1,,,",srsr:1E5,srtbv:3,srtd:1,srtr:500,srvr:"",srwo:!1,ssc:!1,st:3E3,svNB:!1,syntheticConfig:!1,tal:0,tp:"500,50,3",tt:100,tvc:3E3,uam:!1,useNewCookies:!1,uxdce:!1,uxdcw:1500,uxrgce:!0,uxrgcm:"100,25,300,3;100,25,300,3",vcfi:0===h,vs:1,WST:!1,xb:"",xmut:!0,xt:0};a:{var c=z().cfg;f={reportUrl:"dynaTraceMonitor",initializedModules:"",csu:"dtagent",dataDtConfig:"string"===typeof c?c:""};z().cfg=f;0===h&&(f.csu="ruxitagentjs");var e=f.dataDtConfig;e&&-1===p(e,"#CONFIGSTRING")&&(xa(e,f),Ga("domain"),Ga("auto"),Ga("app"),Ha(f));var k=eb("script"),g=fb(k),n=-1===p(f.dataDtConfig||"","#CONFIGSTRING")?f:null;if(0<g)for(a=0;a<g;a++)b:{var b=void 0,q=k[a],c=n;if(q.attributes){var B=f.csu+"_bootstrap.js",e=/.*\/jstag\/.*\/.*\/(.*)_bs(_dbg)?.js$/,D=c,w=q.src,v=w&&w.indexOf(B),E=q.attributes.getNamedItem("data-dtconfig");if(E){var r=w,G=E.value,x={};f.legacy=!0;if(r){var ka=/([a-zA-Z]*)[0-9]{0,4}_([a-zA-Z_0-9]*)_([0-9]+)/g.exec(r);ka&&ka.length&&(x.csu=ka[1],x.featureHash=ka[2],0===h&&(x.agentLocation=r.substr(0,p(r,ka[1])-1),x.buildNumber=ka[3]))}G&&xa(G,x,!0);pb(x.domain)||(x.domainOverride=location.hostname+","+x.domain,delete x.domain);b=x;if(!c)D=b;else if(!b.syntheticConfig){n=b;break b}}b||(b=f);if(v&&0<=v){var L=v+B.length+5;b.app=w.length>L?w.substr(L):"Default%20Application"}else if(w){var M=e.exec(w);M&&(b.app=M[1])}n=D}else n=c}if(n)for(var O in n)n.hasOwnProperty(O)&&(k=O,f[k]=n[k]);f.rx_visitID&&(z().rx_visitID=f.rx_visitID);var da=Fa();try{var T=(n=La)&&n.getItem(da);if(T){var ra=nb(T),y=xa(ra.config||""),C=f.lastModification||"0",V=u((y.lastModification||ra.lastModification||"0").substr(0,13)),ga="string"===typeof C?u(C.substr(0,13)):C;if(!C||V>=ga)if(y.agentname=ra.name,y.agentUri?Ha(y):(y.csu=ra.name,y.featureHash=ra.featureHash),Ia(y,!0),mb(y),lb(y),V>(f.lastModification||0)){var ha=m("auto"),ia=m("legacy");f=ob(y);f.auto=ha;f.legacy=ia}}}catch(Ca){}Ia(f);try{var W=f.ign;if(W&&(new RegExp(W)).test(d.location.href)){document.dT_=d.dT_=void 0;var sa=!1;break a}}catch(Ca){}f.useNewCookies&&0===h&&(ca="rxpc",A="rxsession",wa="rxlatency",hb="x-rxpc",ib="x-rxreferer");sa=!0}if(!sa)return!1;pc();try{Ib=z().disabled||!!ea("dtDisabled")}catch(Ca){}var F;if(!(F=l("agentLocation")))a:{var X=l("agentUri");if(X||document.currentScript){var H=X||document.currentScript.src;if(H){var ja=-1===p(H,"_bs")&&-1===p(H,"_bootstrap")&&-1===p(H,"_complete")?1:2,N=H.lastIndexOf("/");for(sa=0;sa<ja&&-1!==N;sa++)H=H.substr(0,N),N=H.lastIndexOf("/");F=H;break a}}var Y=location.pathname;F=Y.substr(0,Y.lastIndexOf("/"))}Ka=F;U=l("agentname")||l("csu")||(0===h?"ruxitagentjs":"dtagent");"true"===K("dtUseDebugAgent")?0>U.indexOf("dbg")&&(U=l("debugName")||U+"dbg"):U=l("name")||U;if(!m("auto")&&!m("legacy")&&!Ib){var P=l("agentUri")||Ja(l("featureHash")),Q;if(!(Q=m("async")||"complete"===document.readyState)){var R=d.navigator.userAgent,S=R.indexOf("MSIE ");Q=0<S?9>=parseInt(R.substring(S+
5,R.indexOf(".",S)),10):!1}Q?Aa(P,m("async"),void 0,void 0,"dtjsagent"):(document.write('<script id="dtjsagentdw" type="text/javascript" src="'+P+'">\x3c/script>'),document.getElementById("dtjsagentdw")||Aa(P,m("async"),void 0,void 0,"dtjsagent"))}var oa=d.location.href;0===h&&-1!==p(oa,"_DT_RENDERING_")&&(z().RMOD={conf:Gc,ignore:Hc,ID:"_DT_RENDERING_"},Ka&&Aa(Ja("R"),!0,void 0,void 0,"dtjsagent"));K(A)&&(f.cooO=!0);vc();if(0===h){var pa=!!qa(Ya);f.pVO=pa;Ba(Wa)}0===h&&J("hvt")&&(Gb=J("hvt"));za(xb,[1])}catch(Ca){return!1}return!0}(0)){try{delete d.dT_}catch(a){d.dT_=void 0}gb()&&d.console.log("JsAgent initCode initialization failed!")}}})();}).call(this);</script><script defer="defer" src="ruxitagentjs_A27SVfjqrux_10175190917092722.js" type="text/javascript"></script><link href="manifest.json" rel="manifest"/><link href="//i5.walmartimages.com" rel="dns-prefetch"/><script>!function(e,n,t,a,c,s,d,i,m){n.className+=" "+t,s.start=1*new Date,s.end=d=function(){n.className=n.className.replace(RegExp(" ?"+t),"")},(e[a]=e[a]||[]).hide=s,setTimeout(function(){d(),s.end=null},4e3),s.timeout=4e3}(window,document.documentElement,"async-hide","dataLayer",0,{"GTM-D8LF":!0})</script><script>!function(e,t,a,n,g){e[n]=e[n]||[],e[n].push({"gtm.start":(new Date).getTime(),event:"gtm.js"});var m=t.getElementsByTagName(a)[0],r=t.createElement(a);r.async=!0,r.src="https://www.googletagmanager.com/gtm.js?id=GTM-D8LF",m.parentNode.insertBefore(r,m)}(window,document,"script","dataLayer")</script><title>Walmart</title><script>!function(e,o){var a=-1!==(e&&e.location&&e.location.hostname||"").indexOf("bodega"),n=o.createElement("link");o.title=a?"Bodega":"Walmart",n.rel="shortcut icon",n.href=a?"/bodegafavicon.ico":"/favicon.ico",o.getElementsByTagName("head")[0].appendChild(n)}(window,document)</script><link as="script" href="static/js/main.34f8c53c.js" rel="preload"/><link as="script" href="static/js/libs.789530ca.js" rel="preload"/><link as="style" href="static/css/main.34cc6bfb.css" rel="preload"/><link as="font" crossorigin="crossorigin" href="static/media/myriad-pro-light.0afc5626.woff" rel="preload"/><link as="font" crossorigin="crossorigin" href="static/media/myriad-pro-regular.a37eeb21.woff" rel="preload"/><link as="font" crossorigin="crossorigin" href="static/media/myriad-pro-bold.5a0685ef.woff" rel="preload"/><style>.async-hide{opacity:0!important}</style><link href="static/css/main.34cc6bfb.css" rel="stylesheet"/></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div><script src="static/js/libs.789530ca.js" type="text/javascript"></script><script src="static/js/main.34f8c53c.js" type="text/javascript"></script></body></html>

Answer 1

从“网络”标签中找到的一系列API调用中动态检索数据。您可以发出初始请求以收集产品ID及其名称，然后在后续请求中使用这些ID收集价格信息。将所有结果存储在最终字典中。如果订单很重要，您可以考虑使用订购的字典。

您可能希望将其重构为函数调用。

import requests
from bs4 import BeautifulSoup as bs

headers = {'User-Agent':'Mozilla/5.0'}
#make initial request to get product ids and their associated names
data = requests.get('https://www.walmart.com.mx/api/page/search?Ntt=notebook&Nrpp=24&No=0', headers = headers).json()

#build dict of ids and names/titles to later update with info including price
d = {record['records'][0]['attributes']['productId'][0]:{'Title':record['attributes']['product.displayText']} for record in data['contents'][0]['mainArea'][1]['records']}

#build the body to send in the post request
products = [{"productId":record['records'][0]['attributes']['productId'][0],"childSkus":[record['records'][0]['attributes']['productId'][0]]} for record in data['contents'][0]['mainArea'][1]['records']]

#prepare post request to get the price info for the collected product ids
headers = {
    'User-Agent': 'Mozilla/5.0',
    'Content-Type': 'application/json',
    'Accept': 'application/json',
    'Referer': 'https://www.walmart.com.mx/productos?Ntt=notebook'
}

data = '{"productInfos":{"products":'+ str(products) + '}}'
r = requests.post('https://www.walmart.com.mx/api/rest/model/atg/commerce/catalog/ProductCatalogActor/getProductPriceAndPromotions', headers=headers, data=data).json()

#update original dictionary with info including price for latter request using ids
for k,v in r['priceAndPromotions'].items(): 
    d[k]['info'] = v

print(d)

示例输出：

可见页面上的第一项：

print(d['00019199968379'])

无法使用网页抓取提取<p>标签。结果返回null

1 个答案: