#!/usr/bin/env python
# -*- coding:utf8 -*-

from scrapy.cmdline import execute  #导入执行scrapy命令方法
import sys
import os

sys.path.append(os.path.join(os.getcwd())) #给Python解释器，添加模块新路径 ,将main.py文件所在目录添加到Python解释器

execute(['scrapy', 'crawl', 'pach', '--nolog'])  #执行scrapy命令

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
import urllib.response
from lxml import etree
import re


class PachSpider(scrapy.Spider):
    name = 'pach'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']


    def parse(self, response):
        pass

# -*- coding: utf-8 -*-
import scrapy

class PachSpider(scrapy.Spider):
    name = 'pach'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        leir = response.xpath('//a[@class="archive-title"]/text()').extract()  #获取指定标题
        leir2 = response.xpath('//a[@class="archive-title"]/@href ').extract() #获取指定url

        print(response.url)    #获取抓取的rul
        print(response.body)   #获取网页内容
        print(response.body_as_unicode())  #获取网站内容unicode编码

        for i in leir:
            print(i)
        for i in leir2:
            print(i)