手把手教你用.NET Core写爬虫

 private static HtmlParser htmlParser = new HtmlParser();

 private ConcurrentDictionary<string, MovieInfo> \_cdMovieInfo = new ConcurrentDictionary<string, MovieInfo>();

 private void AddToHotMovieList()

        {

 //此操作不阻塞当前其他操作，所以使用Task

 // \_cdMovieInfo 为线程安全字典，存储了当期所有的电影数据

 Task.Factory.StartNew(()=> 

            {

 try

                {

 //通过URL获取HTML

 var htmlDoc = HTTPHelper.GetHTMLByURL("http://www.dy2018.com/");

 //HTML 解析成 IDocument

 var dom = htmlParser.Parse(htmlDoc);

 //从dom中提取所有class='co\_content222'的div标签

 //QuerySelectorAll方法接受 选择器语法 

 var lstDivInfo = dom.QuerySelectorAll("div.co\_content222");

 if (lstDivInfo != null)

                    {

 //前三个DIV为新电影

 foreach (var divInfo in lstDivInfo.Take(3))

                        {

 //获取div中所有的a标签且a标签中含有"/i/"的

 //Contains("/i/") 条件的过滤是因为在测试中发现这一块div中的a标签有可能是广告链接

 divInfo.QuerySelectorAll("a").Where(a => a.GetAttribute("href").Contains("/i/")).ToList().ForEach(

                            a =>

                            {

 //拼接成完整链接

 var onlineURL = "http://www.dy2018.com" + a.GetAttribute("href");

 //看一下是否已经存在于现有数据中

 if (!\_cdMovieInfo.ContainsKey(onlineURL))

                                {

 //获取电影的详细信息

 MovieInfo movieInfo = FillMovieInfoFormWeb(a, onlineURL);

 //下载链接不为空才添加到现有数据

 if (movieInfo.XunLeiDownLoadURLList != null && movieInfo.XunLeiDownLoadURLList.Count != 0)

                                    {

 \_cdMovieInfo.TryAdd(movieInfo.Dy2018OnlineUrl, movieInfo);

                                    }

                                }

                            });

                        }

                    }

                }

 catch(Exception ex)

                {

                }

            });

        }

 private MovieInfo FillMovieInfoFormWeb(AngleSharp.Dom.IElement a, string onlineURL)

        {

 var movieHTML = HTTPHelper.GetHTMLByURL(onlineURL);

 var movieDoc = htmlParser.Parse(movieHTML);

 //http://www.dy2018.com/i/97462.html 分析过程见上，不再赘述

 //电影的详细介绍 在id为Zoom的标签中

 var zoom = movieDoc.GetElementById("Zoom");

 //下载链接在 bgcolor='#fdfddf'的td中，有可能有多个链接

 var lstDownLoadURL = movieDoc.QuerySelectorAll("[bgcolor='#fdfddf']");

 //发布时间 在class='updatetime'的span标签中

 var updatetime = movieDoc.QuerySelector("span.updatetime"); var pubDate = DateTime.Now;

 if(updatetime!=null && !string.IsNullOrEmpty(updatetime.InnerHtml))

            {

 //内容带有“发布时间：”字样，replace成""之后再去转换，转换失败不影响流程

 DateTime.TryParse(updatetime.InnerHtml.Replace("发布时间：", ""), out pubDate);

            }

 var movieInfo = new MovieInfo()

            {

 //InnerHtml中可能还包含font标签，做多一个Replace

 MovieName = a.InnerHtml.Replace("<font color=\"#0c9000\">","").Replace("<font color=\"  #0c9000\">","").Replace("</font>", ""),

 Dy2018OnlineUrl = onlineURL,

 MovieIntro = zoom != null ? WebUtility.HtmlEncode(zoom.InnerHtml) : "暂无介绍...",//可能没有简介，虽然好像不怎么可能

 XunLeiDownLoadURLList = lstDownLoadURL != null ?

 lstDownLoadURL.Select(d => d.FirstElementChild.InnerHtml).ToList() : null,//可能没有下载链接

 PubDate = pubDate,

            };

 return movieInfo;

        }

using System;

using System.Net.Http;

using System.Net.Http.Headers;

using System.Text;

namespace Dy2018Crawler

{

 public class HTTPHelper

    {

 public static HttpClient Client { get; } = new HttpClient();

 public static string GetHTMLByURL(string url)

        {

 try

            {

 System.Net.WebRequest wRequest = System.Net.WebRequest.Create(url);

 wRequest.ContentType = "text/html; charset=gb2312";

 wRequest.Method = "get";

 wRequest.UseDefaultCredentials = true;

 // Get the response instance.

 var task = wRequest.GetResponseAsync();

 System.Net.WebResponse wResp = task.Result;

 System.IO.Stream respStream = wResp.GetResponseStream();

 //dy2018这个网站编码方式是GB2312,

 using (System.IO.StreamReader reader = new System.IO.StreamReader(respStream, Encoding.GetEncoding("GB2312")))

                {

 return reader.ReadToEnd();

                }

            }

 catch (Exception ex)

            {

 Console.WriteLine(ex.ToString());

 return string.Empty;

            }

        }

    }

}

 // This method gets called by the runtime. Use this method to add services to the container.

 public void ConfigureServices(IServiceCollection services)

        {

 // Add framework services.

 services.AddMvc();

 //Add TimedJob services

 services.AddTimedJob();

        }

 public void Configure(IApplicationBuilder app, IHostingEnvironment env, ILoggerFactory loggerFactory)

        {

 //使用TimedJob

 app.UseTimedJob();

 if (env.IsDevelopment())

            {

 app.UseDeveloperExceptionPage();

 app.UseBrowserLink();

            }

 else

            {

 app.UseExceptionHandler("/Home/Error");

            }

 app.UseStaticFiles();

 app.UseMvc(routes =>

            {

 routes.MapRoute(

                    name: "default",

                    template: "{controller=Home}/{action=Index}/{id?}");

            });

 Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);

        }

 public class AutoGetMovieListJob:Job

    {

 // Begin 起始时间；Interval执行时间间隔，单位是毫秒，建议使用以下格式，此处为3小时；SkipWhileExecuting是否等待上一个执行完成，true为等待；

        [Invoke(Begin = "2016-11-29 22:10", Interval = 1000 \* 3600\*3, SkipWhileExecuting =true)]

 public void Run()

        {

 //Job要执行的逻辑代码

 //LogHelper.Info("Start crawling");

 //AddToLatestMovieList(100);

 //AddToHotMovieList();

 //LogHelper.Info("Finish crawling");

        }

   }

 "runtimes": {

 "win7-x64": {},

 "win7-x86": {},

 "osx.10.10-x64": {},

 "osx.10.11-x64": {},

 "ubuntu.14.04-x64": {}

  }

 //"scripts": {

 //  "prepublish": [ "bower install", "dotnet bundle" ],

 //  "postpublish": [ "dotnet publish-iis --publish-folder %publish:OutputPath% --framework %publish:FullTargetFramework%" ]

 //},

"dependencies": {

 "Microsoft.NETCore.App": {

 "version": "1.1.0"

 //"type": "platform"

    },

//还原各种包文件

dotnet restore;

//发布到C:\code\website\Dy2018Crawler文件夹

dotnet publish -r ubuntu.14.04-x64 -c Release -o "C:\code\website\Dy2018Crawler";