# -*- coding: utf-8 -*- from datetime import datetime import time import Queue import csv from threading import Thread from bs4 import BeautifulSoup import requests # capture current time startTime = datetime.now() # create the instance q = Queue.Queue() # specify sitemap to get all site links url = "http://www.valuecityfurniture.com/sitemap.xml" # request sitemap and make the 'soup' r = requests.get(url, timeout=30) data = r.text soup = BeautifulSoup(data.encode('utf-8'), "lxml") def sitemap_parser(soup): # parse sitemap for all links for url in soup.findAll("loc"): q.put(url.text) # add each url to the queue for processing sitemap_parser(soup) # set filename with date + time timestr = time.strftime("%m-%d-%Y-%H-%M-%S") # Set the domain for which the data is for #domain = "americansignaturefurniture-com-custom-page-" domain = "valuecityfurniture-com-" filename = str("redirect-output\\"+domain+"redirect-"+timestr+".csv") # open file where output is written to w = open(filename, 'w') colheader = "Redirect,Status Code,URL,Redirect,Status Code,URL,Redirect,Status Code,URL\n" w.write(colheader) def grab_data_from_queue(): while not q.empty(): # check that the queue isn't empty url = q.get() # get the item from the queue response = requests.get(url) # request the url if response.history: # if there is a redirect print "Request was redirected:" reqredirected = "Request was redirected:" w.write(reqredirected) w.write(",") for resp in response.history: # pull in HTTP response code and url that corresponds; follow redirects print resp.status_code status = str(resp.status_code) w.write(status) w.write(",") print resp.url url = str(resp.url) w.write(url) w.write(",") # find the final destination for the redirected url; store status code (should be 200 OK, 404 ERROR) print "Final destination:" final = "Final destination:" w.write(final) w.write(",") print response.status_code destinationstatus = str(response.status_code) w.write(destinationstatus) w.write(",") print response.url destinationurl = str(response.url) w.write(destinationurl) w.write("\n") q.task_done() # specify that you are done with the item else: # no redirect was found; show the url that was crawled and move on print "Request was not redirected:" noredirect = "Request was not redirected:" w.write(noredirect) w.write(",") print response.status_code noredirectstatus = str(response.status_code) w.write(noredirectstatus) w.write(",") responseurl = str(response.url) print responseurl w.write(responseurl) w.write("\n") q.task_done() # specify that you are done with the item for i in range(8): # aka number of threadtex t1 = Thread(target = grab_data_from_queue) # target is the above function t1.start() # start the thread q.join() # print current time minus the start time print datetime.now()-startTime ''' output Redirect Status Code URL Redirect Status Code URL Redirect Status Code URL Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/sofas Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/loveseats Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/futons Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/chairs-chaises Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/sectionals Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/sleeper-sofas Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/cabinets-storage/storage-ottomans Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/recliners-rockers Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/cabinets-storage/bookcases Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/cabinets-storage Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/seating/ottomans Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/tables Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/cabinets-storage/tv-stands-media-centers Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/tables/coffee-tables Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/tables/sofa-tables Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/fireplaces Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/tables/end-tables Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/rugs Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/pillows Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/wall-art Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/lighting Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/shop-for-leather Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/mirrors Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/shop-by-brand Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/accents/all-accent-pieces Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/shop-for-upholstery Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/shop-all-reclining-sofas Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/shop-all-living-room Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/see-our-collections Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/see-whats-on-sale Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/tables Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/see-our-best-sellers Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/living-room/quick-links/see-our-living-room-packages Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/cabinets-storage/buffets-sideboards Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/tables/dining-tables Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/cabinets-storage Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/cabinets-storage/china-cabinets-curios Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/seating Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/seating/counter-bar-stools Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/tables/dinettes Request was not redirected: 200 Request was not redirected: http://www.valuecityfurniture.com/search/v/dining-room/seating/benches 200 http://www.valuecityfurniture.com/search/v/dining-room/seating/dining-chairs Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/accents Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/accents/lighting Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/accents/wall-art Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/accents/mirrors Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/shop-for-5-piece-dining-rooms Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/accents/all-accent-pieces Request was redirected: 302 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/shop-all-dining-room Final destination: 200 http://www.valuecityfurniture.com/search/v/dining-room Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/shop-by-brand Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/shop-for-7-piece-dining-rooms Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/see-whats-on-sale Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/see-our-best-sellers Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds/full-beds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/dining-room/quick-links/see-our-collections Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds/twin-beds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds/queen-beds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds/king-beds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds/daybeds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/beds/futons Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/cabinets-storage/dressers-chests Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/cabinets-storage Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/pillows Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/cabinets-storage/nightstands Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/lighting Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/bed-linens Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/wall-art Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/rugs Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/all-accent-pieces Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/shop-by-brand Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/accents/mirrors Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/shop-for-5-piece-bedrooms Request was not redirected: 200 Request was not redirected: http://www.valuecityfurniture.com/search/v/mattresses 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/shop-for-6-piece-bedrooms Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/shop-all-bedroom Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/see-whats-on-sale Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/see-our-bedroom-packages Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/see-our-collections Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/bedroom/quick-links/see-our-best-sellers Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/mattress-sets Request was not redirected: 200 Request was not redirected: http://www.valuecityfurniture.com/search/v/mattresses/mattress-sets/queen-mattresses 200 http://www.valuecityfurniture.com/search/v/mattresses/mattress-sets/twin-mattresses Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/mattress-sets/full-mattresses Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/accessories/bed-frames Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/mattress-sets/king-mattresses Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/accessories Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/accessories/pillows Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/accessories/mattress-protectors Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/accessories/bed-linens Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/brands Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/brands/miraclefoam Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/brands/beautyrest Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/brands/icomfort Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/brands/iseries Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/quick-links/shop-all-mattresses Request was not redirected: Request was not redirected: 200 200 http://www.valuecityfurniture.com/search/v/mattresses/quick-links http://www.valuecityfurniture.com/search/v/mattresses/quick-links/see-whats-on-sale Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/mattresses/quick-links/see-our-best-sellers Request was not redirected: Request was not redirected: 200 200 http://www.valuecityfurniture.com/search/v/mattresses/quick-links/see-our-collections http://www.valuecityfurniture.com/search/v/kids/beds Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/kids Request was not redirected: 200 http://www.valuecityfurniture.com/search/v/kids/beds/twin-beds '''
Run
Reset
Share
Import
Link
Embed
Language▼
English
中文
Python Fiddle
Python Cloud IDE
Follow @python_fiddle
Browser Version Not Supported
Due to Python Fiddle's reliance on advanced JavaScript techniques, older browsers might have problems running it correctly. Please download the latest version of your favourite browser.
Chrome 10+
Firefox 4+
Safari 5+
IE 10+
Let me try anyway!
url:
Go
Python Snippet
Stackoverflow Question
