swagger上的接口写入数据库

一、依赖

virtualenv -p python3.6 xx

pip install scrapy

pip install pymysql

二、

1、创建项目和spider1

scrapy startproject scraw_swagger

scrapy genspider spider1 xxx.com (执行之后在项目的spiders目录下会生成一个spider1.py的文件)

以下代码主要实现了将swagger的第一级目录爬下来存在一个叫：interfaces_path的文件下

# -*- coding: utf-8 -*-

import scrapy

import json

from scraw_swagger import settings

class Spider1Spider(scrapy.Spider):

    name = 'spider1'

    allowed_domains = ['xxx.com'']

    scrawl_domain = settings.interface_domain+'/api-docs'

    start_urls = [scrawl_domain]

    def parse(self, response):

        # 调试代码

        # filename = 'mid_link'

        # open(filename, 'wb').write(response.body)

        # /////////

        response = response.body

        response_dict = json.loads(response)

        apis = response_dict['apis']

        n = len(apis)

        temppath = []

        i = 0

        domain = settings.interface_domain+'/api-docs'

        filename = 'interfaces_path'

        file = open(filename, 'w')

        for i in range(0, n):

            subapi = apis[i]

            path = subapi['path']

            path = ','+domain + path

            temppath.append(path)

            file.write(path)

2、创建spider2

scrapy genspider spider2 xxx.com (执行之后在项目的spiders目录下会生成一个spider2.py的文件)

以下代码主要实现了获取interfaces_path的文件下的地址对应的内容

# # -*- coding: utf-8 -*-

import scrapy

from scraw_swagger.items import ScrawSwaggerItem

import json

from scraw_swagger import settings

class Spider2Spider(scrapy.Spider):

    name = 'spider2'

    allowed_domains = ['xxx.com']

    file = open('interfaces_path', 'r')

    file = file.read()

    list_files = []

    files = file.split(',')

    n = len(files)

    for i in range(1, n):

        file = files[i]

        list_files.append(file)

    start_urls = list_files

    def parse(self, response):

        outitem = ScrawSwaggerItem()

        out_interface = []

        out_domain = []

        out_method = []

        out_param_name = []

        out_data_type = []

        out_param_required = []

        # 调试代码

        # filename = response.url.split("/")[-1]

        # open('temp/'+filename, 'wb').write(response.body)

        # ///////

        response = response.body

        response_dict = json.loads(response)

        items = response_dict['apis']

        items_len = len(items)

        for j in range(0, items_len):

            path = items[j]['path']

            # interface组成list

            operations = items[j]['operations'][0]

            method = operations['method']

            parameters = operations['parameters']

            parameters_len = len(parameters)

            param_name = []

            param_required = []

            data_type = []

            for i in range(0, parameters_len):

                name = parameters[i]['name']

                param_name.append(name)

                required = parameters[i]['required']

                param_required.append(required)

                type = parameters[i]['type']

                data_type.append(type)

            out_interface.append(path)

            interface_domain = settings.interface_domain

            out_domain.append(interface_domain)

            out_method.append(method)

            out_data_type.append(data_type)

            out_param_name.append(param_name)

            out_param_required.append(param_required)

        outitem['interface'] = out_interface

        outitem['domain'] = out_domain

        outitem['method'] = out_method

        outitem['param_name'] = out_param_name

        outitem['param_required'] = out_param_required

        outitem['data_type'] = out_data_type

        yield outitem

3、settings.py文件

# -*- coding: utf-8 -*-

# Scrapy settings for scraw_swagger project

#

# For simplicity, this file contains only settings considered important or

# commonly used. You can find more settings consulting the documentation:

#

#     https://doc.scrapy.org/en/latest/topics/settings.html

#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

interface_domain = 'test'

token = 'test'

# 调试代码

# interface_domain = 'http://xxxx..net'

# token = 'xxxxxx'

# ///////////////

BOT_NAME = 'scraw_swagger'

SPIDER_MODULES = ['scraw_swagger.spiders']

NEWSPIDER_MODULE = 'scraw_swagger.spiders'

FEED_EXPORT_ENCODING = 'utf-8'

# Crawl responsibly by identifying yourself (and your website) on the user-agent

#USER_AGENT = 'scraw_swagger (+http://www.yourdomain.com)'

# Obey robots.txt rules

ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)

#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)

# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay

# See also autothrottle settings and docs

#DOWNLOAD_DELAY = 3

# The download delay setting will honor only one of:

#CONCURRENT_REQUESTS_PER_DOMAIN = 16

#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)

#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)

#TELNETCONSOLE_ENABLED = False

# Override the default request headers:

#DEFAULT_REQUEST_HEADERS = {

#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

#   'Accept-Language': 'en',

#}

# Enable or disable spider middlewares

# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html

#SPIDER_MIDDLEWARES = {

#    'scraw_swagger.middlewares.ScrawSwaggerSpiderMiddleware': 543,

#}

# Enable or disable downloader middlewares

# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

#DOWNLOADER_MIDDLEWARES = {

#    'scraw_swagger.middlewares.ScrawSwaggerDownloaderMiddleware': 543,

#}

# Enable or disable extensions

# See https://doc.scrapy.org/en/latest/topics/extensions.html

#EXTENSIONS = {

#    'scrapy.extensions.telnet.TelnetConsole': None,

#}

# Configure item pipelines

# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html

ITEM_PIPELINES = {

   'scraw_swagger.pipelines.ScrawSwaggerPipeline': 300,

   # 'scraw_swagger.pipelines.MysqlTwistedPipline': 200,

}

# Enable and configure the AutoThrottle extension (disabled by default)

# See https://doc.scrapy.org/en/latest/topics/autothrottle.html

#AUTOTHROTTLE_ENABLED = True

# The initial download delay

#AUTOTHROTTLE_START_DELAY = 5

# The maximum download delay to be set in case of high latencies

#AUTOTHROTTLE_MAX_DELAY = 60

# The average number of requests Scrapy should be sending in parallel to

# each remote server

#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0

# Enable showing throttling stats for every response received:

#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)

# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings

#HTTPCACHE_ENABLED = True

#HTTPCACHE_EXPIRATION_SECS = 0

#HTTPCACHE_DIR = 'httpcache'

#HTTPCACHE_IGNORE_HTTP_CODES = []

#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

MYSQL_HOST = ''

MYSQL_DBNAME = ''

MYSQL_USER = ''

MYSQL_PASSWD = ''

MYSQL_PORT = 3306

4、存入数据库。编写pipelines.py文件。提取返回的item，并将对应的字段存入数据库

# -*- coding: utf-8 -*-

import pymysql

from scraw_swagger import settings

from twisted.enterprise import adbapi

import pymysql.cursors

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

class ScrawSwaggerPipeline(object):

    def process_item(self, item, spider):

        try:

            # 插入数据sql

            sql = """

                insert into interfaces (domain, interface, method, param_name ,data_type, param_required)

                VALUES (%s, %s, %s, %s, %s, %s)

            """

            domain = item['domain']

            n = len(domain)

            for i in range(0, n):

                domain = str(item['domain'][i])

                interface = str(item["interface"][i])

                method = str(item["method"][i])

                param_name = str(item["param_name"][i])

                data_type = str(item["data_type"][i])

                param_required = str(item["param_required"][i])

                a = (domain, interface, method, param_name, data_type, param_required)

                self.cursor.execute(sql, a)

                self.connect.commit()

        except Exception as error:

            # 出现错误时打印错误日志

            print(error)

        # self.connect.close()

        return item

    def __init__(self):

        # 连接数据库

        self.connect = pymysql.connect(

            host=settings.MYSQL_HOST,

            db=settings.MYSQL_DBNAME,

            user=settings.MYSQL_USER,

            passwd=settings.MYSQL_PASSWD,

            port=settings.MYSQL_PORT,

            charset='utf8',

            use_unicode=True)

        # 通过cursor执行增删查改

        self.cursor = self.connect.cursor()

巴特西

swagger上的接口写入数据库

最新文章

热门文章