Scrapy, How To Separate Text Within A HTML Tag Element

Code containing my data:

Copy

and then for each one of these td cell, get all text nodes

.//text()

You get something like that:

['\n                ',
 '\n                    ',
 'Century Square',
 '\n                    2 Tampines Central 5',
 '\n                    #01-44-47 Century Square',
 '\n                    Singapore 529509',
 '\n                ',
 '\n                    ',
 'Opening Hours:',
 u'\n                    7am to 12am (Sun-Thu &\xa0PH)',
 u'\n                    24 Hours (Fri & Sat\xa0&',
 '\n                ',
 '\n                    Eve of PH)',
 '\n                    Telephone: 6789 0457',
 '\n            ']

and

['\n                ',
 'Liat Towers',
 '\n                541 Liat towers #01-01',
 '\n                Orchard Road',
 '\n                Singapore 238888',
 'Opening Hours: ',
 '\n                24 hours (Daily)',
 '\n                Telephone: 6737 8036']

Some of these text node have a string representation that is all whitespace, so strip them and look for "Opening hours" and "Telephone" keywords to process lines in a loop:

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
import re
from todo.items import wendyItem

class wendySpider(BaseSpider):
    name = "wendyspider"
    allowed_domains = ["wendys.com.sg"]
    start_urls = ["http://www.wendys.com.sg/outlets.php"]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        cells = hxs.select('//div[@id="menu_list"]//td[@valign="top"][.//span[@class="foodTitle"]]')
        items = []
        for cell in cells:
            item = wendyItem()

            # get all text nodes
            # some lines are blank so .strip() them
            lines = cell.select('.//text()').extract()
            lines = [l.strip() for l in lines if l.strip()]

            # first non-blank line is the place name
            item['name'] = lines.pop(0)

            # for the other lines, check for "Opening hours" and "Telephone"
            # to store lines in correct list container

            address_lines = []
            hours_lines = []
            telephone_lines = []

            opening_hours = False
            telephone = False

            for line in lines:
                if 'Opening Hours' in line:
                    opening_hours = True
                elif 'Telephone' in line:
                    telephone = True
                if telephone:
                    telephone_lines.append(line)
                elif opening_hours:
                    hours_lines.append(line)
                else:
                    address_lines.append(line)

            # last address line is the postal code + town name
            item['address'] = "\n".join(address_lines[:-1])
            item['postal'] = address_lines[-1]

            # ommit "Opening hours" (first element in list)
            item['hours'] = "\n".join(hours_lines[1:])

            item['contact'] = "\n".join(telephone_lines)

            items.append(item)

        return items

Share