Python POST请求不返回HTML，请求启用JavaScript [英] Python POST Request Not Returning HTML, Requesting JavaScript Be Enabled
问题描述
我正在尝试登录我的富国银行帐户并查看我的交易历史记录，以便我可以使用它们来跟踪我的财务状况。如果我可以访问页面的HTML，我可以执行抓取部分。我遇到的问题是到达那里，下面的代码给我带来了很多乱码。
I'm trying to sign in to my Wells Fargo account and scrape my transaction history so that I can use them to track my finances. I am able to do the scraping part if I can get to the HTML of the page. The problem I'm having is getting there and the below code is returning a whole lot of gibberish to me.
####Bring in BeautifulSoup and urllib.
import bs4
import urllib.request
import requests
####Navigate to the website.
url = 'https://connect.secure.wellsfargo.com/auth/login/do'
payload = {"j_username":"USERNAME", "j_password":"PASSWORD"}
r = requests.post(url, payload)
print(r.text)
此代码输出以下内容：
<html><head><meta http-equiv="Pragma" content="no-cache"/>
<meta http-equiv="Expires" content="-1"/>
<meta http-equiv="CacheControl" content="no-cache"/>
<script>
(function(){
var securemsg;
var dosl7_common;
window["bobcmn"] = "1011200000002200000001300000021application/x-www-form-urlencoded3000000088adfa450300000008TSPD_101300000014%2fauth%2flogin%2fdo300000000300000006/TSPD/300000008TSPD_101300000005https3000000b6#sCmnToken#0BC26lnGAWSD9m6NkEoMZy0dIjA7Os6O4oLerWkImSHetiQqPjvoid03xpkXMNwHZ4wUmjd9+FeNk7M7zEe5ESlixC/1O8E7X61l10gL4ddUAhMNR4LaIYlGkq+hckjmRwTXudNvohk90GvOs8Ea9fFIoAAAAAE=#eCmnToken#200000000";
try{(function(){try{var jS,JS,LS=1,oS=1,OS=1,zS=1,S_=1,__=1,i_=1,I_=1,j_=1;for(var L_=0;L_<JS;++L_)LS+=2,oS+=2,OS+=2,zS+=2,S_+=2,__+=2,i_+=2,I_+=2,j_+=3;jS=LS+oS+OS+zS+S_+__+i_+I_+j_;window.i===jS&&(window.i=++jS)}catch(o_){window.i=jS}var O_=window.sdkljshr489=!0;function z_(S){window.sdkljshr489&&S&&(O_=!1);return O_}function Z_(){}z_(window[Z_.name]===Z_);z_("undefined"===window.vodsS0);window.vodsS0=null;z_(/\x3c/.test(function(){return"\x3c"})&/x3d/.test(function(){return"0";"x3d"}));
var s_=/mobi/i.test(navigator.userAgent),Si=+new Date,_i=s_?3E4:3E3;function ii(){return z_(Si+_i<(Si=+new Date))}
(function Ii(){var J=!1;function l(J){for(var l=0;J--;)l+=L(document.documentElement,null);return l}function L(J,l){var Z="vi";l=l||new z;return _S(J,function(J){J.setAttribute("data-"+Z,l.SS());return L(J,l)},null)}function z(){this.O=1;this.L=0;this._=this.O;this.j=null;this.SS=function(){this.j=this.L+this._;if(!isFinite(this.j))return this.reset(),this.SS();this.L=this._;this._=this.j;this.j=null;return this._};this.reset=function(){this.O++;this.L=0;this._=this.O}}var Z=!1;function s(J,l){var L=
document.createElement(J);l=l||document.body;l.appendChild(L);L&&L.style&&(L.style.display="none")}function iS(l,L){L=L||l;var z="|";function s(J){J=J.split(z);var l=[];for(var L=0;L<J.length;++L){var Z="",lS=J[L].split(",");for(var SS=0;SS<lS.length;++SS)Z+=lS[SS][SS];l.push(Z)}return l}var _S=0,IS="datalist,details,embed,figure,hrimg,strong,article,formaddress|audio,blockquote,area,source,input|canvas,form,link,tbase,option,details,article";IS.split(z);IS=s(IS);IS=new RegExp(IS.join(z),"g");while(IS.exec(l))IS=
new RegExp((""+new Date)[8],"g"),J&&(Z=O_),++_S;return L(_S&&1)}function _S(J,l,L){(L=L||Z)&&s("div",J);J=J.children;var z=0;for(var _S in J){L=J[_S];try{L instanceof HTMLElement&&(l(L),++z)}catch(IS){}}return z}iS(Ii,l)})();window.oi={iI:"08c787b5a40180002943d30328de8438de8cc553d459dcd4fc6c4cb17feaa34f085900356d674a1888119e0ea122f11994fc63fbabf471ce1f60053949777f087711d376633d1c30cd2e2f14295017cd8afeedacf0c4783d8b9ec0abec9808a830fa17d4cc351f649688f2b9c98cc0961ddcaf13fb0e7020486252f76f751366cdb10741f04ad6fd"};function _(S){return 753>S}function I(){var S=arguments.length;for(var J=0;J<S;++J)arguments[J]-=38;return String.fromCharCode.apply(String,arguments)}function O(S){return S.toString(36)}(function ji(J){return J?0:ji(J)*ji(J)})(ii());var v;})();}finally{sdkljshr489=false;ie9rgb4=void(0);};
eval((ie9rgb4=function (){var m='function () {/*fQb f_TcC}-di`U_V YU)bWR$+dbikuVe^SdY_^uvkdbikfQb ZCy:Cy<C-!y_C-!y?C-!+V_bufQb <O-}+<O,:C+xx<Ov<Cx-"y_Cx-"y?Cx-#+ZC-<Cx_Cx?C+gY^T_g{Y---ZCssugY^T_g{Y-xxZCvmSQdSXu_OvkgY^T_g{Y-ZCmfQb ?O-gY^T_g{cT[\\ZcXb$()-n}+Ve^SdY_^ jOuCvkgY^T_g{cT[\\ZcXb$()ssCssu?O-n!v+bUdeb^ ?OmVe^SdY_^ JOuvkmjOugY^T_gKJO{^Q]UM---JOv+jOuoe^TUVY^UTo---gY^T_g{f_TcC}v+gY^T_g{f_TcC}-^e\\\\+jOu|Lh#S|{dUcduVe^SdY_^uvkbUdeb^oLh#Somvs|h#T|{dUcduVe^SdY_^uvkbUdeb^o}o+oh#Tomvv+\r\nfQb cO-|]_RY|Y{dUcdu^QfYWQd_b{ecUb1WU^dvyCY-x^Ug 4QdUyOY-cO/#5$*#5#+Ve^SdY_^ YYuvkbUdeb^ jOuCYxOY,uCY-x^Ug 4QdUvvmuVe^SdY_^uvkfQb C-kTUSbi`d*Ve^SdY_^uCvkdbikbUdeb^ :C?>{`QbcUuVe^SdY_^uCvkC-C{c`\\Yduo\\ov+fQb :-oo+V_bufQb \\-}+\\,C{\\U^WdX+xx\\v:x-CdbY^W{Vb_]3XQb3_TUuCK\\Mv+bUdeb^ :muCvvmSQdSXu\\vkmmm+bUdeb^ C-kS_^VYWebQdY_^*C{TUSbi`duo!"#\\#$\\)\'\\))\\!!&\\!}%\\!!(\\!}!\\#$\\%(\\#$\\!!}\\!!!\\#$\\$$\\#$\\!}}\\!}!\\)(\\!!\'\\!}#\\!}#\\!}%\\!!}\\!}#\\#$\\%(\\#$\\!!}\\!!!\\#$\\$$\\#$\\!})\\!!!\\!}}\\!!\'\\!}(\\!}!\\$)\\#$\\%(\\#$\\!}!\\!!}\\)\'\\)(\\!}(\\!}!\\!}}\\#$\\$$\\#$\\!})\\!!!\\!}}\\!!\'\\!}(\\!}!\\%}\\#$\\%(\\#$\\!}!\\!!}\\)\'\\)(\\!}(\\!}!\\!}}\\#$\\$$\\#$\\!})\\!!!\\!}}\\!!\'\\!}(\\!}!\\%!\\#$\\%(\\#$\\!}!\\!!}\\)\'\\)(\\!}(\\!}!\\!}}\\#$\\$$\\#$\\!})\\!!!\\!}}\\!!\'\\!}(\\!}!\\%"\\#$\\%(\\#$\\!}!\\!!}\\)\'\\)(\\!}(\\!}!\\!}}\\#$\\!"%ovmmvuv+\r\ncUSebU]cW-kcZC*Ve^SdY_^uCvkbUdeb^ cUSebU]cWK?u"(()\'vMucUSebU]cW{jYuuOu!&}vy}vyCyOu)""v/}*!vyVe^SdY_^uvkbUdeb^ CdbY^WK9u!$}y!%"y!$)y!$\'y!}%y!$"y!#%y!%"y!}%y!$)y!#(y!#)vMu=QdXK?u"&"}&}!!vMu=QdXK?u!&%}$\'#\'#$vMuvwuOu))#v/#$"*"%&vxuOu""$v/!*}vvruOu")\'v/"%&*#""vvmvK?u)!("#)vMuoovmyjC*Ve^SdY_^uCvkbUdeb^uuCsuOu"%#v/"%%*""}vv,,uOu)$\'v/"!*"$vluCsuOu$(&v/&%"(}*&&%%%vv,,uOu)}!v/%*(vlC..uOu(()v/)*(vsuOu&&"v/&%"(}*\'##\'\'vlC..uOu%%)v/"$*#"vsuOu!&$v/"%%*#"\'vv...uOu!)"vy}vmy9}*Ve^SdY_^uCy:vkV_bufQb \\-ooy<-uOu%&)vy\r\n}v+<,CK?u!")$#))"}%vM+<xxv\\x-CdbY^WKoLe}}&&b_]3Lh&(Qb3_TUoMuCK9u!#\'y!$"y!#%y!%"y!}%y!$)y!#(y!#)y!}#y!%$vMuu<xCK?u!")$#))"}%vMz:vrCK?u!")$#))"}%vMvv+bUdeb^ \\myYZC*Ve^SdY_^uCy:vkbUdeb^ cUSebU]cW{9}uCyCK?u!")$#))"}%vMz:vmy<O*Ve^SdY_^uCy:vkYVuCK?u!")$#))"}%vMn-:K?u!")$#))"}%vMvdXb_g cUSebU]cW{:CuCvycUSebU]cW{:Cu:vyoo+V_bufQb \\-ooy<-uOu\'&#vy}v+<,CK?u!")$#))"}%vM+<xxv\\x-CdbY^WKoLe}}&&bLh&V]Le}}$#XLh&!bLe}}$#_Lh&$UoMuCK9u!#\'y\r\n!$"y!#%y!%"y!}%y!$)y!#(y!#)y!}#y!%$vMu<vN:KoLe}}&#XQbLh$#_TU1doMu<vv+bUdeb^ \\my<C*Ve^SdY_^uCy:vkbUdeb^uuC...uOu""!vy}vvxu:...uOu"")vy}vvsuOu)$&v/"!$\'$(#&$\'*$")$)&\'")%vv...uOu&$#vy}vmyO:*Ve^SdY_^uCy:vkbUdeb^uuC...uOu()(vy}vvz:suOu!}&v/$")$)&\'")%*"!$\'$(#&$\'vv...uOu#\'}vy}vmy_%*Ve^SdY_^uCy:y\\vkdbikYVuCK?u!")$#))"}%vMn-uOu("#v/""*!&vvdXb_goo+YVu:K?u!")$#))"}%vMn-uOu&}&v/(*&vvdXb_goo+fQb <-cUSebU]cW{c_uCv+<KOu()vy}M-cUSebU]cW{jCu<KOu\'(&vy}Mv+<KOu\'(#v/}*!M-cUSebU]cW{jCu<KOu)!(v/\r\n}*!Mv+<KOu)"(v/!*"M-cUSebU]cW{jCu<KOu&\'}vy"Mv+<KOu\'(%v/"*#M-cUSebU]cW{jCu<KOu!\')vy#Mv+fQb j-cUSebU]cW{c_u:vyJ-cUSebU]cW{jCujKOu\'$}vy}Mvyc-cUSebU]cW{jCujKOu(\'}v/}*!MvyYC-u\\/Ou)"&v/"!$\'$(#&$\'*$"$\'})\'"#}$*uOu)&#vy}vv...uOu$\'#vy}v+YVu\\vV_bufQb OC-Ou""!v/!%*!"+OC.-uOu"%#vy}v+OCzzvfQb CC-cUSebU]cW{<CuJ,,uOu#\'%v/$*#vNJ...uOu$#%v/%*#vyJvy\\C-cUSebU]cW{<CuYCy<KYC...uOu""#v/!!*!}vsuOu"}\'v/#*"vMvyc-cUSebU]cW{O:ucyCCN\\CvyYC-cUSebU]cW{O:uYCyOu"}%v/"&%$$#%\'&)*"!$!&$}&("vycC-cUSebU]cW{<Cuc,,uOu\'\'\'vy$vNc...uOu!\')v/\r\n%*"vycvyJC-
*************************************************************
""}"!&2) %%}%"&"6 3%21#225 2"24}2"( "22$%1)" %32#&1}$ 3"4\'661\' 2%4}36#! "34))5(2 %24515!4 )2&$3"2} 53""& \'%&11#)3 }"&4)#}1 )3})}&1) 52}5# \'"}\'&\'(% }%}}%\'!# )%26$1(" 5"2(\'1!$ \'22!"215 }32&!2#( )"4"(5)2 5%4%25}4 \'343562\' }24246"! (&4#4"4$ 6!4$5"$" &(442#6( !641(#&5 (!25!&34 6&2)"&%2 &62}\'\'5! !(2\'$\'\'\' ((}(%15& 66}6&1\'} &&}ç !!}!}2%3 (6&%)566 6(&"15&) &!&2664# !&&336$% 1}}15"\'( 4\'}44"55 $5}$(#%$ #)}#2#3" 1\'&\'"&&! 4}&}!&6\' $)&)$\'$4 #5&5\'\'42 154!&1$1 4)4&%143 $}46}2&& #\'4(#26} 1)2315%# 4522)53% $\'2"36\'6 #}2%665) 24246"!3 31213"(1 %#2#)##} "$2$1#1& 214}#&}% 344\'}&)# %$45%\'") "#4)&\'26 2#&&\'1"5 3$&!$12( %4&(!2}" "1&6"2)$ 2$}225#\' 3#}3(51! %1}%46!2 "4}"56(4oK?u!\'$#))!)(#vMuuu\\ON\r\n9CK9u!#\'y!$"y!#%y!%"y!}%y!$)y!#(y!#)y!}#y!%$vMu\\vvsuOu$&&v/"%%*#\'!vvwuOu\'!!v/)*&vyOu"}!v/(*!!v+\\ON-uOu$!$vyz!v+\\O-=QdXK?u!##($vMu\\Ov+\\On-`QbcU9^duJv/uJCxxycUdDY]U_eduCyuOu!#}vy}vvv*:uvmU\\cU :uvmVe^SdY_^ :uvkfQb C-cUSebU]cWK?u!#"$()#vMu:Ox9u)&vxcxoLe}}#QoxYCx9u)&vx\\OyoLh#}Le}}##ov+T_c\\\'OS_]]_^{9Ju<yCyooyOu"($v/%5#*$}\'#yYYuvvmV_bufQb \\-cUSebU]cW{?CugY^T_g{_Y{Y9yoLe}}#}Lh#!ovy\\-T_c\\\'OS_]]_^{::u\\yn!vy<-cUSebU]cW{:Cu\\KOu&#"vy}Mvy\r\nj-\\KOu!!#v/!*}MyJ-\\KOu$&v/"*!Myc-\\KOu!)%v/#*"MyYC-\\KOu&&#vy$MyOC-\\KOu(%$vy%MyCC-\\KOu$}\'v/&*#MK9u!#\'y!$"y!#%y!%"y!}%y!$)y!#(y!#)y!}#y!%$vMuuOu$$&vy}vvy\\C-1bbQiujvycC-=QdXK?u##")&vMuCCzOCKoLe}}&#Lh&(Le}}&!Lh\'"Le}}$#Lh&VLe}}&$Lh&%Le}}$!Lh\'$oMuuOu&%$vy}vvxuOu!)"v/!*}vyjvyJC-uOu\'"%vy}v+JC,j+JCxxv\\CKJCM-OC+fQb JC-uOu)"}vy}vy:Oy9Cy\\O+cUdDY]U_eduCyuOu"\'\'vy}vvmv+\r\nVe^SdY_^ OuCvkbUdeb^ \'%#.CmVe^SdY_^ 9uvkfQb C-QbWe]U^dc{\\U^WdX+V_bufQb :-}+:,C+xx:vQbWe]U^dcK:Mz-#(+bUdeb^ CdbY^W{Vb_]3XQb3_TU{Q``\\iuCdbY^WyQbWe]U^dcvmVe^SdY_^ ?uCvkbUdeb^ C{d_CdbY^Wu#&vmuVe^SdY_^ ZYu:vkbUdeb^ :/}*ZYu:vwZYu:vmvuYYuvv+fQb f+mvuv+mVY^Q\\\\ikcT[\\ZcXb$()-VQ\\cU+YU)bWR$-f_YTu}v+m+*/;}'.slice(15,-4);for(var i=0,c=8,j=0,l,r='';l=m.charCodeAt(i);++i)c=String.fromCharCode(l<33||l>=126?l:(93+l-((-76E-3+''+({}).a).slice(7).charCodeAt(j%'1')))%93+33),r+=c,j-=c.indexOf('\x0d');return r;})());
})();
</script>
<noscript>Please enable JavaScript to view the page content.</noscript>
</head><body>
</body></html>
我为可怕的格式化道歉，但我不知道如何处理它。另外，为了长度的缘故，我在中间删除了一个用星号代替的大的任意部分。
I apologize for the hideous formatting but I didn't know what to do with it. Also, I removed a large, arbitrary portion in the middle that I replaced with the asterisks for the sake of length.
对我而言，我看到的关键是请启用JavaScript以查看页面内容。这个输出实际上是JS吗？我如何处理Python的任何内容？我根本不知道这告诉我什么，我非常感谢你提供的任何帮助。
To me, the key thing I'm seeing is "Please enable JavaScript to view the page content." Is this output actually JS and how do I handle whatever it is with Python? I simply have no clue what this is telling me and I greatly appreciate any help you can provide.
谢谢。
推荐答案
我知道已经花了很多时间，但我可以在这里给出一些关闭。您所看到的是由 F5 Networks，Inc。的优秀人员销售的机器人失败代码，旨在防止天真webcrawler和scrapers无法访问使用它的网站。
I know that a great deal of time has passed on this, but I can give some closure here. What you're seeing is bot-defeat code sold by the good fellows at F5 Networks, Inc., designed to prevent naive webcrawlers and scrapers from being able to access sites that use it.
简而言之，这是一个混淆的Javascript，它通过一系列迭代步骤计算一个值，这些步骤执行各种特定于浏览器的操作Javascript功能，并利用一些相当粗鲁的Javascript语言行为。该值将作为cookie发送回Wells Fargo，并作为导航所需的部分网络表单。只使用无头浏览器不会削减它 - 计算中有一些技巧专门用于对抗无头浏览器和与它们一起使用的Javascript引擎。错过任何技巧都不会导致任何失败;相反，它会以一种让你难以辨别错过的方式抛弃最终结果。
Briefly, this is obfuscated Javascript which calculates a value through a series of iterative steps which exercise various browser-specific Javascript capabilities, and makes use of some rather rude Javascript language behavior. That value is sent back to Wells Fargo as cookies and part of the webforms required for navigation. Just using a headless browser is not going to cut it - there are a few tricks in the calculation designed specifically to counter headless browsers and the Javascript engines that work with them. Missing any of the tricks will not cause any sort of failure; instead, it will just throw off the end result in a way which makes it difficult for you to tell what you missed.
从理论上讲，它可以解读代码并用您选择的语言模拟所有计算;我知道数据汇总公司写的一个成功的对策，但该代码不公开供公众阅读。或者，您可以在JS解释器中找出正确执行它所需的内容。我不记得所有的细节，但它比看起来容易。您无需对整个工程进行逆向工程，只需在正确的环境中运行即可。您需要一个虚拟窗口对象和更多的虚拟对象代码所需的其他代码，例如您环境中的navigator.userAgent，以及其他可能的东西。
It is, in theory, possible to decipher the code and emulate all the calculations in the language of your choice; I know of a successful countermeasure written by a data aggregation company, but the code is not open for public perusal. Alternately, you could figure out what you need to correctly execute it as-is in a JS interpreter. I don't remember all the details, but it's easier than it looks. You don't need to reverse engineer the whole thing, you just need to run it in the right environment. You need a dummy window object and more dummies for whatever else the code is looking for like navigator.userAgent in your environment, plus maybe other things.
出于实际目的，它是写一个对策可能不值得。如果你是一个组织，要求被列入白名单。
For practical purposes, it's probably not worth it to write a countermeasure. Ask to be whitelisted if you're an organization.
如果你对挑战感兴趣，这里有一个（也许是显而易见的）起点 - 长串的乱码 eval（（ie9rgb4 = function（）{var m ='function（） ... .slice .. part是加密代码。紧接着for循环包含字符转换。你可以复制在该循环中完成的操作来破译第一级混淆。通过普通浏览器登录到站点，调试器处于活动状态，观察请求并发送cookie以了解您正在寻找的最终目标，并尝试将其与您看到的JS代码相关联。
If you are interested in the challenge, here is a (perhaps obvious) starting point - the long string of gibberish in the eval((ie9rgb4=function (){var m='function () ... .slice ... portion is ciphered code. The immediately following for loop contains character transformations. You can replicate the operation being done in that loop to decipher the first level of obfuscation. Log on to the site through your normal browser with a debugger active, observe the requests and cookies sent for an idea of the final goal you're looking for, and try to correlate that with the JS code you see.
您还可以找到以下映射在某些时候有用的值：
You may also find the following mapping of values useful at some point:
{"$$$", "7"},
{"$$$$", "f"},
{"$$$_", "e"},
{"$$_", "6"},
{"$$_$", "d"},
{"$$__", "c"},
{"$_", "constructor"},
{"$_$", "5"},
{"$_$$", "b"},
{"$_$_", "a"},
{"$__", "4"},
{"$__$", "9"},
{"$___", "8"},
{"_", "u"},
{"_$", "o"},
{"_$$", "3"},
{"_$_", "2"},
{"__", "t"},
{"__$", "1"},
{"___", "0"}
这篇关于Python POST请求不返回HTML，请求启用JavaScript的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
