python爬取漫画

抓取漫画的网址是：sf互动传媒

抓取漫画的由来也是看了知乎上有人说用爬取漫画，然后自己也玩玩

首页中每个漫画的url是类似这样存储的：

<tr>

     <td height="30" align="center" bgcolor="#FFFFFF">

       <a href="http://comic.sfacg.com/HTML/KOL/" target="_blank">K.O.I 偶像之王</a>

     </td>

</tr>

然后用lxml通过cssselect(tr>td>a)将能用到的解析出来，然后解析出来会有很多其他的页面的url和信息，然后我是通过url中包含"/mh/"或者是"/HTML/"进行过滤的

比较蠢的办法了

然后通过对象，将过滤出来的漫画的url和漫画的名字保存在一个这样的类中，然后通过列表进行存储

class Cartoon():

    url = None

    name = None

然后用随便一个漫画作为例子：勇者赫鲁库

漫画一共很多章，并且所有章的信息都在如下标签中包含

<ul class="serialise_list Blue_link2">....</ul>

然后通过BS将每一章的信息进行存储，然后可以看到其完整的url是：http://comic.sfacg.com/HTML/YZHLK/096/

然后每一章会有很多的page，并且每一章的内容是ajax进行加载的，然后从检查－>网络中可以看到有一个这样的请求

然后请求的response里面包含本章的所有图片，然后只需将在每一章的页面中将.js接口找到，然后将本章的图片信息存储，然后就可以得到本章的图片信息，然后存储到本地

# -*- coding: utf-8 -*-

import re

import urllib

import urllib2

import os

import stat

import itertools

import re

import sys

import requests

import json

import time

import socket

import urlparse

import csv

import random

from datetime import datetime, timedelta

import lxml.html

from zipfile import ZipFile

from StringIO import StringIO

from downloader import Downloader

from bs4 import BeautifulSoup

from HTMLParser import HTMLParser

from itertools import product

import sys

reload(sys)

sys.setdefaultencoding('utf8')

URL = 'http://comic.sfacg.com'

picture = 'http://coldpic.sfacg.com'

class Cartoon():

    url = None

    name = None

def download(url, user_agent='wswp', num_try=2):

    headers = {'User_agent': user_agent}

    request = urllib2.Request(url, headers=headers)

    try:

        html = urllib2.urlopen(request).read()

    except urllib2.URLError as e:

        print 'Download error', e.reason

        html = None

        if num_try > 0:

            if hasattr(e, 'code') and 500 <= e.code < 600:

                return download(url, user_agent, num_try - 1)

            elif e.code == 403:

                return None

    return html

def get_section_url(url):

    html = download(url)

    if html == None:

        return None

    soup = BeautifulSoup(html, "html.parser")

    results = soup.find_all(name='ul', attrs={'class': 'serialise_list Blue_link2'})

    res = r'<a.*?href="([^"]*)".*?>([\S\s]*?)</a>'

    links = re.findall(res, str(results),re.S | re.M)

    return links

def get_section_page(url):

    html = download(url)

    if html == None:

        return None

    soup = BeautifulSoup(html, "html.parser")

    results = soup.find_all(name='script', attrs={'type': 'text/javascript'})

    tt = len(results)

    js = results[tt-1]

    mm = js.get('src')

    if mm == None:

        result = soup.find_all(name='script', attrs={'language': 'javascript'})

        js1 = result[1]

        mm = js1.get('src')

    html1 = download(URL+mm)

    list = html1.split(';')

    List = []

    for each in list:

        if 'picAy[' in each:

            src = each.split('=')

            List.append(picture+src[1][2:-1])

    return List

def download_cartoon(url, cartoon_name,Section,num):

    path = "自己定义的路径"+cartoon_name

    if not os.path.exists(path):

        os.mkdir(path)

    path = path + "/"+Section

    if not os.path.exists(path):

        os.mkdir(path)

    content = requests.get(url).content

    with open(path + '/' + str(num) + '.jpg', 'wb') as f:

        f.write(content)

    print "Downloading cartoon_name " + path + str(num)+ "下载完成"

    f.close()

if __name__ == '__main__':

    cartoon_list = []

    html = download(URL)

    tree = lxml.html.fromstring(html)

    results = tree.cssselect('tr > td > a')

    for each in results:

        ti = each.get('href')

        if '/mh/' in ti or '/HTML/' in ti:

            if each.text_content() != "":

                cartoon = Cartoon()

                cartoon.url = each.get('href')

                cartoon.name = each.text_content().replace(' ','')

                cartoon_list.append(cartoon)

    for each in cartoon_list:

        print each.url

        print each.name

        links = get_section_url(each.url)

        links = list(reversed(links))

        section = 0

        for link in links:

            ul = URL + link[0]

            List = []

            List = get_section_page(ul)

            section = section + 1

            Section = r'第'+ str(section) + r'章'

            num = 1

            for mm in List:

                #print mm

                download_cartoon(mm,each.name,Section,num)

                num = num + 1

            print each.name + Section + "下载完成"+str(num-1)+"张"

巴特西

python爬取漫画

最新文章

热门文章