我有一个python蜘蛛的下面的问题，蜘蛛提取所有正确的数据，但它把所有的数据放在一起，我想分组对应于网站上的帖子的不同项目。
这是蜘蛛的代码。
import scrapy
from scrapy import *
from auInd.items import AuindItem
class auInSpider(scrapy.Spider):
name = "auiS"
start_urls = ["https://au.indeed.com/jobs?as_and=&as_phr=&as_any=&as_not=&as_ttl=&as_cmp=&jt=all&st=&salary=&radius=0&l=Australia&fromage=any&limit=50&sort=&psf=advsrch",
]
def parse(self, response):
yield {
'title':response.xpath('//h2[@class="jobtitle"]//text()').extract(),
'company': response.xpath('//span[@class="company"]//text()').extract(),
'location': response.xpath('//span[@itemprop="jobLocation"]//text()').extract(),
'sumary': response.xpath('//span[@class="summary"]//text()').extract(),
'jobType': response.xpath('//td[@class="snip"]/text()').extract(),
'date': response.xpath('//span[@class="date"]/text()').extract(),
'salary': response.xpath('//nobr/text()').extract()
}
这是输出为.json
[
{"salary": [" -\n ", "$36.23 an hour", "$27.14 an hour","$23,869 - $57,376 a year", "$67,248 - $73,635 a year", "$969 a week", "$49,105 - $58,558 a year", "$910 - $921 a week", "$24 an hour", "$43,777 - $45,699 a year", "$27.32 an hour", "About"], "title": ["\n", "Team Member - New Store - Exhibition St Melbourne", "\n", "\n", "Sales Assistant", "\n", "\n", "Visitor Services Officer", "\n", "\n", "Supervisor Mainline", "\n", "\n", "Administrative Assistant", "\n", "\n", "Replenishment Team Member - Woolworths Featherbrook", "\n", "\n", "Casual Retail Team Member", "\n", "\n", "Services Officer - Benalla", "\n", "\n", "Data Entry Job Openings - Quick Start.", "\n", "\n", "Liquor Team Member - Dan Murphy's Echuca", "\n", "\n", "Team Member - Coles Nightfill Albany Orana", "\n", "\n", "VIC Horsham - Part Time", "\n", "\n", "Library Assistant - Casual", "\n", "\n", "Cafe Assistant", "\n", "\n", "High Pay Data Entry Jobs - Apply Today.", "\n", "\n", "Looking For Data Entry Workers - High Pay.", "\n", "\n", "Administrative Assistant", "\n", "\n", "Produce Team Member - Woolworths Darwin City", "\n", "\n", "Data Entry Workers Wanted In Melbourne.", "\n", "\n", "Customer Service Team Member - Woolworths Kambalda", "\n", "\n", "Registrar, Warrnambool Court Complex", "\n", "\n", "Customer Service Officer", "\n", "\n", "Parking Officer", "\n", "\n", "Liquor Team Member - BWS Thrift Park", "\n", "\n", "Retail Customer Service Officer", "\n", "\n", "Receptionist", "\n", "\n", "Customer Service Officer", "\n", "\n", "Facility & Grounds Officer - Casual - Karoonda MPS", "\n", "\n", "Team Member - Coles Grocery Tumut", "\n", "\n", "Sales Support Officer", "\n", "\n", "Retail Assistant - MITTAGONG", "\n", "\n", "National Visual Merchandising Manager", "\n", "\n", "Administration Assistant", "\n", "\n", "Crop Worker", "\n", "\n", "Casual Office Team Member", "\n", "\n", "Burnie Crew Member", "\n", "\n", "CCTV Operator", "\n", "\n", "Food Service Assistant", "\n", "\n", "Shop Assistant", "\n", "\n", "Medical Receptionist - Prahran", "\n", "\n", "Sales Assistant", "\n", "\n", "Replenishment Team Member - Woolworths Derby", "\n", "\n", "Team Member - Woolworths Fuel Bell Post", "\n", "\n", "Customer Teller", "\n", "\n", "Commercial Manager-Supply Chain (VIC)", "\n", "\n", "Sales Team Member - Tamworth", "\n", "\n", "Receptionist", "\n", "\n", "Customer Service Representative", "\n", "\n", "Customer Assistant - Dan Murphy's Armidale", "\n", "\n", "Team Member - Coles Nightfill - Coles Westcourt", "\n"], "company": ["\n ", "\n ", "\n Officeworks Superstores", "\n", "\n ", "\n City of Greater Bendigo", "\n", "\n ", "\n ", "\n Rio Tinto", "\n", "\n ", "\n ", "\n Monash Health", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n ", "\n Supercheap Auto", "\n", "\n ", "\n V/Line", "\n", "\n ", "\n ", "\n Dan Murphy's", "\n", "\n ", "\n ", "\n Coles", "\n", "\n ", "\n ", "\n Sanity Entertainment", "\n", "\n ", "\n Griffith City Council", "\n", "\n ", "\n ", "\n Department of Corrective Services", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n Court Services Victoria", "\n", "\n ", "\n ", "\n NSW Government", "\n", "\n ", "\n ", "\n Orange City Council", "\n", "\n ", "\n ", "\n BWS", "\n", "\n ", "\n ", "\n Australia Post", "\n", "\n ", "\n Frasers Property Australia", "\n", "\n ", "\n Clarence Valley Council", "\n", "\n ", "\n ", "\n Government of South Australia", "\n", "\n ", "\n ", "\n Coles", "\n", "\n ", "\n ", "\n Elders", "\n", "\n ", "\n ", "\n ALDI Australia", "\n", "\n ", "\n ", "\n Kmart", "\n", "\n ", "\n The Disability Trust", "\n", "\n ", "\n ", "\n Super Amart", "\n", "\n ", "\n ", "\n Hungry Jack's", "\n", "\n ", "\n ", "\n Veolia", "\n", "\n ", "\n Tasmanian Government", "\n", "\n ", "\n ", "\n I-MED Radiology Network", "\n", "\n ", "\n ", "\n The Just Group", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n ", "\n Fuel", "\n", "\n ", "\n ", "\n NAB - National Australia Bank", "\n", "\n ", "\n ", "\n Kmart", "\n", "\n ", "\n ", "\n Fantastic Furniture", "\n", "\n ", "\n Tenixsol", "\n", "\n ", "\n ", "\n Dan Murphy's", "\n", "\n ", "\n ", "\n Coles", "\n"], "sumary": ["\nTeam member discounts,. The new Exhibition St store is due to open in April with the team to commence in their roles in March....", "\nDuties will include but not limited to. A busy local Bakery is seeking a motivated, self driven person to join their team in a part time role....", "\nBookings and retail). Birth Certificate or Extract and any formal registration of name changes such as a Marriage....", "\nAs part of our Mine of the Future\u2122 programme, we will implement AutoHaul\u00ae technology - the first long distance heavy haulage deployment of automated trains,...", "\nBe part of a team supporting excellent education and training for pre vocational doctors with Monash Health;...", "\nAbility to work flexible retail hours. You will work to fill and maintain the presentation of stock during and after trading hours as well as provide great...", "\nShare your passion for Automotive retail with our customers and team. Fuel your passion and drive your career at Australasia's leading automotive retail brand....", "\nThe selection process may include aptitude assessments, a behavioural event interview, medical check and a national probity (police) check....", "\nPlease see the description. We are looking for part time workers who are motivated and enthusiastic. Data Entry Job Openings - Quick Start....", "\nAnd because Dan Murphy\u2019s is part of Woolworths Group, you\u2019ll be part of Australia\u2019s largest retail group. You will also be responsible for creating a store with...", "\nFocused on ensuring the store is fully stocked for our customers each morning. Development and you'll enjoy great benefits like team member discounts, hot....", "\nPrevious retail experience required. Sanity Entertainment now has over 150 stores Australia wide and will continue to grow....", "\nThe successful applicant will require a higher school certificate or relevant experience in customer service and/or library operations....", "\nBe well presented. If you are interested in the position and fulfil the above criteria please forward your CV and a cover letter....", "\nHigh Pay Data Entry Jobs - Apply Today. Please see the description. This is an amazing job opportunity for all who want to to earn full-time income....", "\nPlease see the description. Looking For Data Entry Workers - High Pay. If you are looking for a perfect job then this opportunity is right for you....", "\nAdministrative support includes, but is not limited to, the maintenance of all administrative records and filing, switchboard and reception duties, official and...", "\nDo you have a passion for retail and customer service? Excited about being able to work on a checkout to serve customers at the end of their shopping experience...", "\nPlease see the description. Must be able to work weekday and weekends, No experience is required. Data Entry Workers Wanted In Melbourne....", "\nAre you passionate about retail and providing exceptional customer service? For many of our customers, you are the face of Woolworths and you have the ability...", "\nApplications close Thursday 2 February 2017. You are only required to address the Key Selection Criteria listed in the Application Form....", "\nRelevant tertiary qualifications or equivalent experience desirable. Closing date Thursday 2 February 2017 [11:59PM]....", "\nTo be successful in this role you will need a current NSW Class C Drivers Licence as well as the ability to achieve a satisfactory criminal background check....", "\nWe are much more than a liquor store though, we are part of Australia's biggest retail group, Woolworths Limited, and have the most trusted food, liquor and...", "\nRetail & Consumer Products. Appropriate knowledge and awareness of Occupational Health and Safety relevant to the position....", "\nReporting to the Personal Assistant to the General Manager, the Receptionist will be responsible for being the face and voice of our Victorian office along with...", "\nPreferred applicants for this position will be required to undergo a Criminal History Check. Monday 6 February 2017, at 4:00 PM (NSW time)....", "\nDuties may include rubbish collection, minor carpentry, painting, plumbing, mechanical and grounds maintenance....", "\nOr, as a Receiving Dock Team Member , you'll play a key part in making sure our stores are fully stocked. Focused on ensuring the store is fully stocked for our...", "\nApplications close 5 February 2017. At Elders, we want our people to reflect the communities we operate in, and encourage applications from a diverse range of...", "\nA Retail Assistant at ALDI needs to be able to thrive in a fast paced, team environment and always be willing to go the extra mile for their customers and team....", "\nWe are transforming the look and feel of our stores and have 210 stores already open throughout Australia and New Zealand....", "\nPlease address the selection criteria in the position description. For more information download the position description by clicking here....", "\nWe get hundreds of applications for many different jobs. Harvest & pack produce, pruning, planting etc. Start mid March to April 2017....", "\nAvailability to work weekend and late night trade shifts. We are looking for passionate people who jump out of bed in the morning to enjoy the interaction with...", "\nOpportunity to complete nationally accredited qualifications in Retail. With over 400 stores in Australia and more than 17,000 employees we are an iconic...", "\nConduit and report on condition and features certificate. Applications close on Friday 10 February 2017. Minimum 3-5 year experience in a....", "\nDedicated to improving the health and wellbeing of our community. Production, plating service, ware washing, and cleaning duties in the provision....", "\nTill operator, Cleaning. Salary: Not Specified Work Type: Casual position Tenancy: Temporary position Hours: 38 Positions: 1 How to Apply: Please", "\nThe I-MED Network is Australia\u2019s largest private diagnostic imaging network offering a comprehensive range of services including CT, MRI, ultrasound, general x...", "\nWe offer a fun, supportive environment, as well as training & development with opportunities to advance your retail career....", "\nAbility to work flexible retail hours. You will work to fill and maintain the presentation of stock during and after trading hours as well as provide great...", "\nBe a part of Australia\u2019s largest retail group. Ensure the health and safety of all store associates and customers....", "\nWith over 5,000 staff across our network of over 750 retail branches, agencies and kiosks our people are the face of NAB and our focus is providing our...", "\nWe have over 200 stores already open throughout Australia and New Zealand, we employ over 30,000 team members and we have another 50 new stores in the pipeline!...", "\nEnjoy great in store competitions which includes winning trips overseas and new carsStaff discounts on all of our group retail brands for you and your family....", "\nAs the receptionist at our busy and industry leading occupational therapy firm, you will be the first point of contact for liaising with patients and...", "\nSuccessful candidates are required to complete and pass a National Police Check and an Infringement (outstanding matters) Check....", "\nAnd because Dan Murphy\u2019s is part of Woolworths Group, you\u2019ll be part of Australia\u2019s largest retail group. You will also be responsible for creating a store with...", "\nFocused on ensuring the store is fully stocked for our customers each morning. Development and you'll enjoy great benefits like team member discounts, hot...."], "location": ["Victoria", "Lakes Entrance VIC", "Bendigo VIC", "Cape Lambert WA", "Victoria", "Victoria", "Victoria", "Benalla VIC", "The Rocks NSW", "Echuca VIC", "Albany WA", "Horsham VIC", "Griffith NSW", "Derby WA", "Hobart TAS", "Brisbane QLD", "Australia", "Darwin NT", "Melbourne VIC", "Kambalda WA", "Warrnambool VIC", "Far West NSW", "Orange NSW", "Victoria", "Narrogin WA", "Victoria", "Grafton NSW", "Karoonda SA", "Tumut NSW", "Ayr QLD", "Mittagong NSW", "Victoria", "Bega NSW", "Atherton QLD", "Mackay QLD", "Burnie TAS", "Victoria", "Latrobe TAS", "Maningrida NT", "Victoria", "Port Macquarie NSW", "Derby WA", "Victoria", "Portland VIC", "Victoria", "Tamworth NSW", "Paddington QLD", "Victoria", "Armidale NSW", "Cairns QLD"], "date": ["1 day ago", "16 hours ago", "4 days ago", "1 day ago", "3 hours ago", "1 day ago", "1 day ago", "4 days ago", "10 hours ago", "4 days ago", "1 day ago", "4 days ago", "6 days ago", "10 hours ago", "10 hours ago", "10 hours ago", "1 day ago", "17 hours ago", "10 hours ago", "5 days ago", "5 days ago", "4 days ago", "4 days ago", "4 days ago", "1 day ago", "15 hours ago", "5 days ago", "4 days ago", "5 days ago", "5 days ago", "7 hours ago", "12 hours ago", "7 days ago", "10 hours ago", "13 hours ago", "4 days ago", "15 hours ago", "4 days ago", "11 days ago", "6 days ago", "10 hours ago", "17 hours ago", "11 days ago", "14 hours ago", "6 days ago", "5 days ago", "10 hours ago", "20 hours ago", "17 hours ago", "21 hours ago"], "jobType": ["\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n", "\n", "\n\n", "\n\n", "\n\n"]},
{"salary": [" -\n ", "$36.23 an hour", "$27.14 an hour", "$23,869 - $57,376 a year", "$67,248 - $73,635 a year", "$969 a week", "$49,105 - $58,558 a year", "$910 - $921 a week", "$24 an hour", "$43,777 - $45,699 a year", "$27.32 an hour", "About"], "title": ["\n", "Team Member - New Store - Exhibition St Melbourne", "\n", "\n", "Sales Assistant", "\n", "\n", "Visitor Services Officer", "\n", "\n", "Supervisor Mainline", "\n", "\n", "Administrative Assistant", "\n", "\n", "Replenishment Team Member - Woolworths Featherbrook", "\n", "\n", "Casual Retail Team Member", "\n", "\n", "Services Officer - Benalla", "\n", "\n", "Data Entry Job Openings - Quick Start.", "\n", "\n", "Liquor Team Member - Dan Murphy's Echuca", "\n", "\n", "Team Member - Coles Nightfill Albany Orana", "\n", "\n", "VIC Horsham - Part Time", "\n", "\n", "Library Assistant - Casual", "\n", "\n", "Cafe Assistant", "\n", "\n", "High Pay Data Entry Jobs - Apply Today.", "\n", "\n", "Looking For Data Entry Workers - High Pay.", "\n", "\n", "Administrative Assistant", "\n", "\n", "Produce Team Member - Woolworths Darwin City", "\n", "\n", "Data Entry Workers Wanted In Melbourne.", "\n", "\n", "Customer Service Team Member - Woolworths Kambalda", "\n", "\n", "Registrar, Warrnambool Court Complex", "\n", "\n", "Customer Service Officer", "\n", "\n", "Parking Officer", "\n", "\n", "Liquor Team Member - BWS Thrift Park", "\n", "\n", "Retail Customer Service Officer", "\n", "\n", "Receptionist", "\n", "\n", "Customer Service Officer", "\n", "\n", "Facility & Grounds Officer - Casual - Karoonda MPS", "\n", "\n", "Team Member - Coles Grocery Tumut", "\n", "\n", "Sales Support Officer", "\n", "\n", "Retail Assistant - MITTAGONG", "\n", "\n", "National Visual Merchandising Manager", "\n", "\n", "Administration Assistant", "\n", "\n", "Crop Worker", "\n", "\n", "Casual Office Team Member", "\n", "\n", "Burnie Crew Member", "\n", "\n", "CCTV Operator", "\n", "\n", "Food Service Assistant", "\n", "\n", "Shop Assistant", "\n", "\n", "Medical Receptionist - Prahran", "\n", "\n", "Sales Assistant", "\n", "\n", "Replenishment Team Member - Woolworths Derby", "\n", "\n", "Team Member - Woolworths Fuel Bell Post", "\n", "\n", "Customer Teller", "\n", "\n", "Commercial Manager-Supply Chain (VIC)", "\n", "\n", "Sales Team Member - Tamworth", "\n", "\n", "Receptionist", "\n", "\n", "Customer Service Representative", "\n", "\n", "Customer Assistant - Dan Murphy's Armidale", "\n", "\n", "Team Member - Coles Nightfill - Coles Westcourt", "\n"], "company": ["\n ", "\n ", "\n Officeworks Superstores", "\n", "\n ", "\n City of Greater Bendigo", "\n", "\n ", "\n ", "\n Rio Tinto", "\n", "\n ", "\n ", "\n Monash Health", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n ", "\n Supercheap Auto", "\n", "\n ", "\n V/Line", "\n", "\n ", "\n ", "\n Dan Murphy's", "\n", "\n ", "\n ", "\n Coles", "\n", "\n ", "\n ", "\n Sanity Entertainment", "\n", "\n ", "\n Griffith City Council", "\n", "\n ", "\n ", "\n Department of Corrective Services", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n Court Services Victoria", "\n", "\n ", "\n ", "\n NSW Government", "\n", "\n ", "\n ", "\n Orange City Council", "\n", "\n ", "\n ", "\n BWS", "\n", "\n ", "\n ", "\n Australia Post", "\n", "\n ", "\n Frasers Property Australia", "\n", "\n ", "\n Clarence Valley Council", "\n", "\n ", "\n ", "\n Government of South Australia", "\n", "\n ", "\n ", "\n Coles", "\n", "\n ", "\n ", "\n Elders", "\n", "\n ", "\n ", "\n ALDI Australia", "\n", "\n ", "\n ", "\n Kmart", "\n", "\n ", "\n The Disability Trust", "\n", "\n ", "\n ", "\n Super Amart", "\n", "\n ", "\n ", "\n Hungry Jack's", "\n", "\n ", "\n ", "\n Veolia", "\n", "\n ", "\n Tasmanian Government", "\n", "\n ", "\n ", "\n I-MED Radiology Network", "\n", "\n ", "\n ", "\n The Just Group", "\n", "\n ", "\n ", "\n Woolworths Supermarkets", "\n", "\n ", "\n ", "\n Fuel", "\n", "\n ", "\n ", "\n NAB - National Australia Bank", "\n", "\n ", "\n ", "\n Kmart", "\n", "\n ", "\n ", "\n Fantastic Furniture", "\n", "\n ", "\n Tenixsol", "\n", "\n ", "\n ", "\n Dan Murphy's", "\n", "\n ", "\n ", "\n Coles", "\n"]},
]
但是，我正在寻找的输出更像是：
[
{"salary":["..."], "title":["..."], "summary": ["..."]}
]
我希望每个帖子都被提取为不同的字典。欢迎所有帮助，意见和想法。谢谢。
答案 0 :(得分：2)
您需要遍历每个搜索结果并为每个搜索结果生成一个项目：
def parse(self, response):
for result in response.css(".result"):
yield {
'title':result.xpath('.//h2[@class="jobtitle"]//text()').extract_first(),
'company': result.xpath('.//span[@class="company"]//text()').extract_first(),
'location': result.xpath('.//span[@itemprop="jobLocation"]//text()').extract_first(),
'sumary': result.xpath('.//span[@class="summary"]//text()').extract_first(),
'jobType': result.xpath('.//td[@class="snip"]/text()').extract_first(),
'date': result.xpath('.//span[@class="date"]/text()').extract_first(),
'salary': result.xpath('.//nobr/text()').extract_first()
}
注意使用result.xpath()代替response.xpath()，XPath表达式中的前置点（使它们在当前＆＃34;结果＆＃34;元素的范围内工作）和{ {1}}代替extract_first()。