C#制作网盘搜索工具(简单的爬虫）
最近学习C#编程，在网上发现一篇winform下制作百度网盘搜索器的文章，故而下载源码学习一二。无奈原博所用的网址失效，故而自己改写了网址和相关源代码，也进行了实现。因为初学，接触的知识较多，为免忘记，进行整理复习。
1.知识点：
思路：主要是利用HttpWebRequest，HttpWebResponse进行http模拟请求，然后利用HtmlAgilityPack+XPath语法对html dom进行元素获取，将截取到的相关内容在datagridview中展示，最后利用process.start()方法进行点击访问。
2.具体实现：
2.1关于请求头的获取：
本例子使用网址为：http://www.pansoso.com/
分析上述网址的请求头进行模拟：
查看具体请求头信息：
根据获取的request url分析出其请求地址的规律为：所搜索的关键字：hello直接利用get方法添加到了url的最后，其中页数规律为hello_1,hello_2。。。（每页十条记录）
2.2关于结果的获取：
结果的获取，直接利用对response网页的分析截取关键信息即可。
3.代码实现：
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
namespace 百度网盘资源搜索
{
class HttpHelper
{
static readonly string urlTemplate = "http://www.pansoso.com/zh/{0}";
public static string Requset(string key)
{
string url = string.Format(urlTemplate, key);
//Console.WriteLine(url);
HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(url);
httpRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36";
httpRequest.Host = "www.pansoso.com";
httpRequest.Referer = "http://www.pansoso.com/zh/" + Uri.EscapeUriString(key);
try
{
HttpWebResponse httpResponse = (HttpWebResponse)httpRequest.GetResponse();
Stream s = httpResponse.GetResponseStream();
StreamReader sr = new StreamReader(s);
string jsonString = sr.ReadToEnd();
//Console.WriteLine(jsonString);
//string jsonProcessed = null;
//if ((jsonProcessed = JsonPreProcessing(jsonString)) != null)
//{
// SearchResult searchResult = UtilityClass.GetObject<SearchResult>(jsonProcessed);
// return searchResult;
//}
return jsonString;
}
catch
{
return null;
}
}
public static SearchResult dodata(string str)
{
SearchResult searchResult = UtilityClass.GetObject<SearchResult>(str);
return searchResult;
}
//if (doc.DocumentNode.SelectNodes("//comment()") != null)
//{
// foreach (var commet in doc.DocumentNode.SelectNodes("//comment"))
// {
// commet.Remove();
// }
//}
public static string JsonPreProcessing(string jsonString)
{
int startIndex = jsonString.IndexOf("(");
if (startIndex > 0)
{
string json = jsonString.Substring(startIndex + 1);
return "{\"resources\":" + json.Remove(json.Length - 3) + "}";
}
else
{
return null;
}
}
}
}Utility.Class
using System;
using System.Collections.Generic;
using System.IO;
//using System.Linq;
using System.Runtime.Serialization.Json;
using System.Text;
namespace 百度网盘资源搜索
{
class UtilityClass
{
public static T GetObject<T>(string json)
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(T));
MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(json));
T obj = (T)serializer.ReadObject(ms);
return obj;
}
}
}JSontoObject.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace 百度网盘资源搜索
{
public class SearchResult
{
public BDWPResource[] resources { get; set; }
}
public class BDWPResource
{
public string title { get; set; }
public string content { get; set; }
public string unescapedUrl { get; set; }
}
}
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Threading;
using System.Diagnostics;
namespace 百度网盘资源搜索
{//主窗体
public partial class FrmMain : Form
{
bool isSearch = true;
string url = "http://www.pansoso.com";
public FrmMain()
{
InitializeComponent();
}
private void btnSearch_Click(object sender, EventArgs e)
{
string key = this.txtKey.Text;
if (!string.IsNullOrEmpty(key))
{
this.dataGridView1.Rows.Clear();
this.lblResult.Text = "0";
this.pgsBar.Value = 0;
this.btnSearch.Text = "正在搜索";
this.btnSearch.Enabled = false;
this.btnStop.Enabled = true;
Thread thread = new Thread(() =>
{
for (int i = 1; i < 11; i ++)//共取得10页网页数据
{
if (isSearch)
{
gethtml(HttpHelper.Requset(key+"_"+i));
//gethtml(HttpHelper.Requset(key));
//if(textBox1.Text!=null)
//{
// string name=textBox1.Text;
// SearchResult sr= HttpHelper.dodata(name);
// if (sr != null)
// {
// foreach (BDWPResource resource in sr.resources)
// {
// BindResource(resource);
// }
// }
// }
// webBrowser1.DocumentText = HttpHelper.Requset(key);
// Navigate to HTML document string
//webBrowser1.Navigate(HttpHelper.Requset(key));
// SearchResult sr = HttpHelper.Requset(key);
}
else break;
}
//搜索完成
SearchOver();
});
thread.IsBackground = true;
thread.Start();
}
}
public void gethtml(string docs)
{
try
{
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(docs);
if (doc.DocumentNode.SelectNodes("//script") != null)
{
foreach (var script in doc.DocumentNode.SelectNodes("//script"))
{
script.Remove();
}
HtmlAgilityPack.HtmlNodeCollection hrefList = doc.DocumentNode.SelectNodes(".//h2/a[@href]");
HtmlAgilityPack.HtmlNodeCollection list2 = doc.DocumentNode.SelectNodes(".//div[@class='des']");
HtmlAgilityPack.HtmlNodeCollection list3 = doc.DocumentNode.SelectNodes(".//h2/a[@href]");
if (hrefList != null && list2 != null && list3 != null)
{
for (int i = 0; i < list2.Count; i++)
{
string url1 = url + list3[i].Attributes["href"].Value;
string json = "title:" + hrefList[i].InnerText + "content:" + list2[i].InnerText + "unescapedUrl:" +"【"+url1+"】" ;
// Process.Start(url1);
SearchOver1(json);
this.Invoke(new Action<string, string, string>((tle, ctt, url3) =>
{
this.dataGridView1.Rows.Add(tle, ctt, url3);
this.lblResult.Text = (Int32.Parse(this.lblResult.Text) + 1).ToString();
if (this.pgsBar.Value < this.pgsBar.Maximum)
{
this.pgsBar.Value++;
}
}), hrefList[i].InnerText,list2[i].InnerText, url1);
}
}
}
}
catch (Exception)
{
MessageBox.Show("该关键字没有收录资源！！！");
}
}
//if (doc.DocumentNode.SelectNodes("//style") != null)
//{
// foreach (var style in doc.DocumentNode.SelectNodes("style"))
// {
// style.Remove();
// }
//}
private void BindResource(BDWPResource resource)
{
string title = resource.title.Replace("</b>", "").Replace("<b>", "");
string content = resource.content.Replace("</b>", "").Replace("<b>", "");
this.Invoke(new Action<string, string, string>((tle, ctt, url) =>
{
this.dataGridView1.Rows.Add(tle, ctt, url);
this.lblResult.Text = (Int32.Parse(this.lblResult.Text) + 1).ToString();
if (this.pgsBar.Value < this.pgsBar.Maximum)
{
this.pgsBar.Value++;
}
}), title, content, resource.unescapedUrl);
}
private void SearchOver()
{
this.Invoke(new Action(() =>
{
this.btnSearch.Text = "开始搜索";
this.btnSearch.Enabled = true;
this.btnStop.Enabled = false;
this.isSearch = true;
}));
}
public void SearchOver1(string str)
{
this.Invoke(new Action(() =>
{
this.richTextBox1.Text += str + System.Environment.NewLine;
}));
}
private void dataGridView1_RowPostPaint(object sender, DataGridViewRowPostPaintEventArgs e)
{
SolidBrush b = new SolidBrush(this.dataGridView1.RowHeadersDefaultCellStyle.ForeColor);
e.Graphics.DrawString((e.RowIndex + 1).ToString(System.Globalization.CultureInfo.CurrentUICulture), this.dataGridView1.DefaultCellStyle.Font, b, e.RowBounds.Location.X + 20, e.RowBounds.Location.Y + 6);
e.Graphics.FillRectangle(Brushes.White, new Rectangle(new Point(e.RowBounds.Location.X + 2, e.RowBounds.Location.Y + 2), new Size(20, 20)));//隐藏每行前面的图标
}
//打开网页链接
private void dataGridView1_CellDoubleClick(object sender, DataGridViewCellEventArgs e)
{
if (e.RowIndex > -1)
{
string url = this.dataGridView1.Rows[e.RowIndex].Cells[2].Value.ToString();
Process.Start(url);//进行打开浏览器的方法。
}
}
private void btnStop_Click(object sender, EventArgs e)
{
isSearch = false;
this.btnSearch.Enabled = true;
}
private void richTextBox1_LinkClicked(object sender, LinkClickedEventArgs e)
{
System.Diagnostics.Process.Start(e.LinkText);
}
}
}4.效果实现：
手机扫一扫
移动阅读更方便
你可能感兴趣的文章