c#网络爬虫-采集51job数据（html分析基于HtmlAgilityPack）

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.Threading;
namespace DataCollect
{
    public partial class MainForm : Form
    {
        public MainForm()
        {
            InitializeComponent();
        }

        List<Job> list = new List<Job>();//定义用于存放数据的集合
        CancellationTokenSource cts = new CancellationTokenSource();
        private void btnCollect_Click(object sender, EventArgs e)
        {
            cts.Cancel();
            //数据采集方法
            Task task1 = new Task(Collect);
            task1.Start();
           // Collect();
        }
        /// <summary>
        /// 数据采集
        /// </summary>
        private void Collect()
        {
            int index = 1;
            int pageCount = 1;
            do
            {
                var htmlDocument = DataLoad(index);
                if (index==1)
                {
                   HtmlNode hnNode= htmlDocument.DocumentNode.SelectSingleNode("//input[@id='hidTotalPage']");
                   pageCount=int.Parse(hnNode.Attributes["value"].Value);
                    this.listView1.Invoke(new Action(() => {
                        this.lblTotalPages.Text = pageCount.ToString();
                    }));
                    SetLabel(this.lblTotalPages, pageCount.ToString());
                }
                Thread.Sleep(2000);
                    //信息解析
                //var htmlDocument = DataLoad(1);//弱类型赋值

                HtmlNode hn = htmlDocument.DocumentNode.SelectSingleNode("//div[@class='el title']");
                //递归（循环检测还有没有兄弟元素）
                GetNode(hn.NextSibling);
                index  ;
                SetLabel(this.lblCurrentNo, index.ToString());
                SetLabel(this.lblItems, (50 * pageCount).ToString());
                
                //数据组合

            } while (index<pageCount);
        }
        private void SetLabel(Label lbl,string value)
        {
            lbl.Invoke(new Action(() =>
            {
                lbl.Text = value;
            }));
        }
        private void GetNode(HtmlNode parentNode)
        {
            Thread.Sleep(100);
            if(parentNode!=null)
            {
                if (parentNode.ChildNodes.Count > 0)
                {
                    Job job = new Job();
                    job.Position = parentNode.ChildNodes[1].InnerText.Replace("\r\n", "").Trim();//只取标签中间的内容
                    job.Company = parentNode.ChildNodes[3].InnerText;
                    job.Address = parentNode.ChildNodes[5].InnerText;
                    job.Salary = parentNode.ChildNodes[7].InnerText;
                    job.CreateTime = parentNode.ChildNodes[9].InnerText;
                    list.Add(job);//添加到集合

                    this.listView1.Invoke(new Action(()=> {
                        //创建控件对象
                        ListViewItem lvi = new ListViewItem();
                        lvi.Text = job.Position;
                        lvi.SubItems.AddRange(new string[] {job.Company, job.Address, job.Salary, job.CreateTime });
                        //数据添加到界面的控件
                        this.listView1.Items.Add(lvi);
                    }));
                    
                }
            }
            //判断到分页阶段
           if(parentNode.NextSibling.Attributes["class"] != null &&
                parentNode.NextSibling.Attributes["class"].Value == "dw_page")
            {
                return;//结束
            }


            GetNode(parentNode.NextSibling);//调用自己
        }
        private HtmlAgilityPack.HtmlDocument DataLoad(int _index)
        {
            HtmlWeb htmlWeb = new HtmlWeb();//创建Html解析对象
            htmlWeb.OverrideEncoding = Encoding.GetEncoding("gbk");
            string url = $"https://search.51job.com/list/120200,000000,0000,00,9,99,.net,2,{_index}.html";
            return htmlWeb.Load(url);
        }

        private void btnStop_Click(object sender, EventArgs e)
        {
            cts.Cancel();
        }
    }
}

资源下载

此资源下载价格为3D币（VIP免费），请先登录

资源文件列表

DataCollect/.vs/DataCollect/v15/.suo , 34304
DataCollect/.vs/DataCollect/v15/Server/sqlite3/db.lock , 0
DataCollect/.vs/DataCollect/v15/Server/sqlite3/storage.ide , 606208
DataCollect/.vs/DataCollect/v15/Server/sqlite3/storage.ide-shm , 32768
DataCollect/.vs/DataCollect/v15/Server/sqlite3/storage.ide-wal , 4128272
DataCollect/DataCollect/App.config , 189
DataCollect/DataCollect/bin/Debug/DataCollect.exe , 14336
DataCollect/DataCollect/bin/Debug/DataCollect.exe.config , 189
DataCollect/DataCollect/bin/Debug/DataCollect.pdb , 38400
DataCollect/DataCollect/bin/Debug/HtmlAgilityPack.dll , 162816
DataCollect/DataCollect/bin/Debug/HtmlAgilityPack.pdb , 298496
DataCollect/DataCollect/bin/Debug/HtmlAgilityPack.xml , 156495
DataCollect/DataCollect/DataCollect.csproj , 3795
DataCollect/DataCollect/Job.cs , 421
DataCollect/DataCollect/MainForm.cs , 4476
DataCollect/DataCollect/MainForm.Designer.cs , 9154
DataCollect/DataCollect/MainForm.resx , 5817
DataCollect/DataCollect/obj/Debug/DataCollect.csproj.CopyComplete , 0
DataCollect/DataCollect/obj/Debug/DataCollect.csproj.CoreCompileInputs.cache , 42
DataCollect/DataCollect/obj/Debug/DataCollect.csproj.FileListAbsolute.txt , 1472
DataCollect/DataCollect/obj/Debug/DataCollect.csproj.GenerateResource.cache , 1015
DataCollect/DataCollect/obj/Debug/DataCollect.csprojAssemblyReference.cache , 45404
DataCollect/DataCollect/obj/Debug/DataCollect.exe , 14336
DataCollect/DataCollect/obj/Debug/DataCollect.MainForm.resources , 180
DataCollect/DataCollect/obj/Debug/DataCollect.pdb , 38400
DataCollect/DataCollect/obj/Debug/DataCollect.Properties.Resources.resources , 180
DataCollect/DataCollect/obj/Debug/DesignTimeResolveAssemblyReferences.cache , 1407
DataCollect/DataCollect/obj/Debug/DesignTimeResolveAssemblyReferencesInput.cache , 7214
DataCollect/DataCollect/obj/Debug/TemporaryGeneratedFile_036C0B5B-1481-4323-8D20-8F5ADCB23D92.cs , 0
DataCollect/DataCollect/obj/Debug/TemporaryGeneratedFile_5937a670-0e60-4077-877b-f7221da3dda1.cs , 0
DataCollect/DataCollect/obj/Debug/TemporaryGeneratedFile_E7A71F73-0F8D-4B9B-B56E-8E70B10BC5D3.cs , 0
DataCollect/DataCollect/Program.cs , 526
DataCollect/DataCollect/Properties/AssemblyInfo.cs , 1320
DataCollect/DataCollect/Properties/Resources.Designer.cs , 2835
DataCollect/DataCollect/Properties/Resources.resx , 5612
DataCollect/DataCollect/Properties/Settings.Designer.cs , 1098
DataCollect/DataCollect/Properties/Settings.settings , 249
DataCollect/DataCollect.sln , 1131