j bear

j bear

  • NA
  • 1
  • 2k

Code help please

Aug 6 2010 11:02 AM
Hi,
I am trying to scrape a page for an url, then visit that url and scrape more info from that page. My code works in that it scrapes the url, but i cant make it follow the url and scrape again... can someone help me out? code below, thanks if you can help!


    WebClient objWebClient = new WebClient();
        string url = "http://www.bing.com/search?q=" + TextBox1.Text;
        byte[] aRequestHTML;
                aRequestHTML = objWebClient.DownloadData(url);
        UTF8Encoding utf8 = new UTF8Encoding();
        string myString = null;
        myString = utf8.GetString(aRequestHTML);




        Regex r = new Regex("<div class=\"sb_tlst\"><h3><a href=\"(.*?)\" onmousedown=\"");
        MatchCollection mcl = r.Matches(myString);

    
        ArrayList aray = new ArrayList();

        foreach (Match ml in mcl)
        {
           
            foreach (Group g in ml.Groups)
            {
                string bminor = g.Value;
                bminor = string.Format("<a href='{0}'>{0}</a>", g.Value);
                aray.Add(bminor);
             
            }
        }



        WebClient objWebClient2 = new WebClient();
        string bminor2 = myString;
        byte[] aaRequestHTML;
        aaRequestHTML = objWebClient2.DownloadData(bminor2);
    
        string myString2 = null;
        myString2 = utf8.GetString(aaRequestHTML);


        Regex d = new Regex("<meta name=\"Description\" content=\"(.*?)\">");
        MatchCollection mcl3 = d.Matches(myString2);


        ArrayList aray2 = new ArrayList();

        foreach (Match ml3 in mcl3)
        {

            foreach (Group descrip in ml3.Groups)
            {
                string des = descrip.Value;
                des = string.Format("{0}", descrip.Value);
                aray2.Add(des);

            }
        }



        DataTable dt = new DataTable();
        DataRow dr = null;
        dt.Columns.Add(new DataColumn("Column1", typeof(string)));
        for (int i = 0; i < aray.Count; i++)
        {
            dr = dt.NewRow();
            dr["Column1"] = aray[i].ToString();
            dt.Rows.Add(dr);
        }

        dt.Columns.Add(new DataColumn("Column2", typeof(string)));
        for (int i = 0; i < aray2.Count; i++)
        {
            dr = dt.NewRow();
            dr["Column2"] = aray2[i].ToString();
            dt.Rows.Add(dr);
        }

       
        DataGrid1.DataSource = dt;
        DataGrid1.DataBind();

Answers (1)