C#-HttpWebRequest获取数据下一页问题 [英] C# - HttpWebRequest Get Data Next Page(s) Issue
本文介绍了C#-HttpWebRequest获取数据下一页问题的处理方法，对大家解决问题具有一定的参考价值，需要的朋友们下面随着小编来一起学习吧！
问题描述
我尝试使用httpwebrequest从网站中提取数据.我无法从第一个页面"中提取数据，但是当我尝试将数据获取到下一页或上一页时，出现错误.
Im try to extract data from a website with httpwebrequest. Im able to extract data from the first "Page" but when i try to get data to the next or previous page i get a error.
要查看Post参数，我使用Fiddler来比较我的代码和网站中的Post Request标头.
To see the Post parameters i use Fiddler to compare the Post Request Header from my Code and the website.
我的实际C#代码如下:
My actual C# code its this:
string URI = "https://www.citius.mj.pt/Portal/consultas/ConsultaVendaAnuncios.aspx";
CookieContainer cc = new CookieContainer();
HttpWebRequest h = (HttpWebRequest)WebRequest.Create(URI);
h.CookieContainer = cc;
HttpWebResponse hr = (HttpWebResponse)h.GetResponse();
using (var s = hr.GetResponseStream())
{
using (var r = new StreamReader(s, Encoding.UTF8))
{
string html = r.ReadToEnd();
Match mVS = Regex.Match(html, "id=\"__VIEWSTATE\" value=\"(.*?)\"", RegexOptions.Singleline);
Match mEV = Regex.Match(html, "id=\"__EVENTVALIDATION\" value=\"(.*?)\"", RegexOptions.Singleline);
///////////////////////////////////////SEARCH///////////////////////////////////////////////////
Dictionary<string, string> param = new Dictionary<string, string>();
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$ScriptManager1|ctl00$ContentPlaceHolder1$btnSearch");
param.Add("__EVENTTARGET", "");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
param.Add("__VIEWSTATE", mVS.Groups[1].Value);
param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__ASYNCPOST", "true");
param.Add("ctl00$ContentPlaceHolder1$btnSearch", "Pesquisar");
string post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
var data = Encoding.UTF8.GetBytes(post);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
int x = 1;
////////////////////////////////////////////NEXT PAGE//////////////////////////////////////////
param.Clear();
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$upResultados|ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__EVENTTARGET", "ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
param.Add("__VIEWSTATE", mVS.Groups[1].Value);
param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("__ASYNCPOST", "true");
post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
data = Encoding.UTF8.GetBytes(post);
request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
response = (HttpWebResponse)request.GetResponse();
responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
int y = 1;
}
}
当我尝试获取下一页数据时得到的结果如下: 提琴手的结果
The result i get when i trying to get nextpage data its: Result from fiddler
比较我的代码和网站调用之间的标头，唯一的区别是我看到的是"_EVENTEVALUATION".
Comparing the header between calls from my code and the website the only difference i see its the "_EVENTEVALUATION".
我在做什么错了?
推荐答案
如我所建议的，您应该拆分响应字符串.
As I suggest, you should split response string.
string URI = "https://www.citius.mj.pt/Portal/consultas/ConsultaVendaAnuncios.aspx";
CookieContainer cc = new CookieContainer();
HttpWebRequest h = (HttpWebRequest)WebRequest.Create(URI);
h.CookieContainer = cc;
HttpWebResponse hr = (HttpWebResponse)h.GetResponse();
using (var s = hr.GetResponseStream())
{
using (var r = new StreamReader(s, Encoding.UTF8))
{
string html = r.ReadToEnd();
Match mVS = Regex.Match(html, "id=\"__VIEWSTATE\" value=\"(.*?)\"", RegexOptions.Singleline);
Match mEV = Regex.Match(html, "id=\"__EVENTVALIDATION\" value=\"(.*?)\"", RegexOptions.Singleline);
///////////////////////////////////////SEARCH///////////////////////////////////////////////////
Dictionary<string, string> param = new Dictionary<string, string>();
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$ScriptManager1|ctl00$ContentPlaceHolder1$btnSearch");
param.Add("__EVENTTARGET", "");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
param.Add("__VIEWSTATE", mVS.Groups[1].Value);
param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__ASYNCPOST", "true");
param.Add("ctl00$ContentPlaceHolder1$btnSearch", "Pesquisar");
string post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
var data = Encoding.UTF8.GetBytes(post);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
File.WriteAllText("D:\\tmp1.txt", responseString);
////////////////////////////////////////////NEXT PAGE//////////////////////////////////////////
param.Clear();
string[] split = responseString.Split('|');
for (int i = 0; i < split.Length; i++)
{
if (split[i] == "__VIEWSTATE") param.Add("__VIEWSTATE", split[i + 1]);
if (split[i] == "__EVENTVALIDATION") param.Add("__EVENTVALIDATION", split[i + 1]);
}
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$upResultados|ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__EVENTTARGET", "ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
//param.Add("__VIEWSTATE", mVS.Groups[1].Value);
//param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("__ASYNCPOST", "true");
post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
data = Encoding.UTF8.GetBytes(post);
request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
response = (HttpWebResponse)request.GetResponse();
responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
File.WriteAllText("D:\\tmp2.txt", responseString);
int y = 1;
}
}
这篇关于C#-HttpWebRequest获取数据下一页问题的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
查看全文
