Hi,
I am trying to scrape a page for an url, then visit that url and scrape more info from that page. My code works in that it scrapes the url, but i cant make it follow the url and scrape again... can someone help me out? code below, thanks if you can help!
WebClient objWebClient = new WebClient();
string url = "http://www.bing.com/search?q=" + TextBox1.Text;
byte[] aRequestHTML;
aRequestHTML = objWebClient.DownloadData(url);
UTF8Encoding utf8 = new UTF8Encoding();
string myString = null;
myString = utf8.GetString(aRequestHTML);
Regex r = new Regex("<div class=\"sb_tlst\"><h3><a href=\"(.*?)\" onmousedown=\"");
MatchCollection mcl = r.Matches(myString);
ArrayList aray = new ArrayList();
foreach (Match ml in mcl)
{
foreach (Group g in ml.Groups)
{
string bminor = g.Value;
bminor = string.Format("<a href='{0}'>{0}</a>", g.Value);
aray.Add(bminor);
}
}
WebClient objWebClient2 = new WebClient();
string bminor2 = myString;
byte[] aaRequestHTML;
aaRequestHTML = objWebClient2.DownloadData(bminor2);
string myString2 = null;
myString2 = utf8.GetString(aaRequestHTML);
Regex d = new Regex("<meta name=\"Description\" content=\"(.*?)\">");
MatchCollection mcl3 = d.Matches(myString2);
ArrayList aray2 = new ArrayList();
foreach (Match ml3 in mcl3)
{
foreach (Group descrip in ml3.Groups)
{
string des = descrip.Value;
des = string.Format("{0}", descrip.Value);
aray2.Add(des);
}
}
DataTable dt = new DataTable();
DataRow dr = null;
dt.Columns.Add(new DataColumn("Column1", typeof(string)));
for (int i = 0; i < aray.Count; i++)
{
dr = dt.NewRow();
dr["Column1"] = aray[i].ToString();
dt.Rows.Add(dr);
}
dt.Columns.Add(new DataColumn("Column2", typeof(string)));
for (int i = 0; i < aray2.Count; i++)
{
dr = dt.NewRow();
dr["Column2"] = aray2[i].ToString();
dt.Rows.Add(dr);
}
DataGrid1.DataSource = dt;
DataGrid1.DataBind();