利用pycurl做GETMETHOD
import urllib2import pycurlimport cStringIO as _StringIOimport sysimport shutilimport lxml.html as Himport threadingimport StorageClient as scimport lame# http transfer limitsaccept_type = "*/*"connection_timeout = 500 timeout = 800low_speed = 200low_speed_time = 120max_size = 20485760 # 10MBdef get_curl(user_agent="MSIE"): "initialize curl handle" dev_null = _StringIO.StringIO() curl_handle = pycurl.Curl() curl_handle.setopt(pycurl.FOLLOWLOCATION, 1) curl_handle.setopt(pycurl.MAXREDIRS, 5) curl_handle.setopt(pycurl.CONNECTTIMEOUT, connection_timeout) curl_handle.setopt(pycurl.TIMEOUT, timeout) curl_handle.setopt(pycurl.NOSIGNAL, 1) curl_handle.setopt(pycurl.LOW_SPEED_LIMIT, 100) curl_handle.setopt(pycurl.LOW_SPEED_TIME, low_speed_time) curl_handle.setopt(pycurl.HTTPHEADER, ["User-Agent: %s" % "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", accept_type]) curl_handle.setopt(pycurl.MAXFILESIZE, max_size) curl_handle.setopt(pycurl.COOKIEFILE, 'cookies.txt') curl_handle.setopt(pycurl.COOKIEJAR, 'cookies.txt') curl_handle.setopt(pycurl.WRITEFUNCTION, dev_null.write) return curl_handledef curl_fetch(curl_handle, url): "retrieve url, return the content, http code, time, effective url" fp = _StringIO.StringIO() curl_handle.setopt(pycurl.URL, url) curl_handle.setopt(pycurl.WRITEFUNCTION, fp.write) # perform the transfer try: curl_handle.perform() except pycurl.error, e: print e return (-1,0,0) content_type = curl_handle.getinfo(pycurl.CONTENT_TYPE) print curl_handle.getinfo(pycurl.CONTENT_TYPE) print curl_handle.getinfo(pycurl.HTTP_CODE) return (0,fp.getvalue(), content_type)
页:
[1]