Python PycURL 网络编程实例
在使用urllib的时候经常会死掉,以前debug过,是没有设置 timing out 所以超时后就会死掉。
PycURL是curl的python库,虽然有些curl的功能没有实现,但是还是很强劲的。
curl是非常强劲的一个工具,
google内部用它来 debug GDATA API. Using cURL to interact with Google data services
可以去 http://pycurl.sourceforge.net/ 下载最新的PycURL。
简单的PycURL例子
import pycurl import StringIO url = "http://www.google.com/" crl = pycurl.Curl() crl.setopt(pycurl.VERBOSE,1) crl.setopt(pycurl.FOLLOWLOCATION, 1) crl.setopt(pycurl.MAXREDIRS, 5) crl.fp = StringIO.StringIO() crl.setopt(pycurl.URL, url) crl.setopt(crl.WRITEFUNCTION, crl.fp.write) crl.perform() print crl.fp.getvalue()
PycURL 自动处理cookie
import pycurl import StringIO url = "http://www.google.com/" crl = pycurl.Curl() crl.setopt(pycurl.VERBOSE,1) crl.setopt(pycurl.FOLLOWLOCATION, 1) crl.setopt(pycurl.MAXREDIRS, 5) crl.fp = StringIO.StringIO() crl.setopt(pycurl.URL, url) crl.setopt(crl.WRITEFUNCTION, crl.fp.write) # Option -b/--cookie Cookie string or file to read cookies from # Note: must be a string, not a file object. crl.setopt(pycurl.COOKIEFILE, "cookie_file_name") # Option -c/--cookie-jar Write cookies to this file after operation # Note: must be a string, not a file object. crl.setopt(pycurl.COOKIEJAR, "cookie_file_name") crl.perform() print crl.fp.getvalue()
PycURL 实现POST方法
import pycurl import StringIO import urllib url = "http://www.google.com/" post_data_dic = {"name":"value"} crl = pycurl.Curl() crl.setopt(pycurl.VERBOSE,1) crl.setopt(pycurl.FOLLOWLOCATION, 1) crl.setopt(pycurl.MAXREDIRS, 5) #crl.setopt(pycurl.AUTOREFERER,1) crl.setopt(pycurl.CONNECTTIMEOUT, 60) crl.setopt(pycurl.TIMEOUT, 300) #crl.setopt(pycurl.PROXY,proxy) crl.setopt(pycurl.HTTPPROXYTUNNEL,1) #crl.setopt(pycurl.NOSIGNAL, 1) crl.fp = StringIO.StringIO() crl.setopt(pycurl.USERAGENT, "dhgu hoho") # Option -d/--data HTTP POST data crl.setopt(crl.POSTFIELDS, urllib.urlencode(post_data_dic)) crl.setopt(pycurl.URL, url) crl.setopt(crl.WRITEFUNCTION, crl.fp.write) crl.perform() print crl.fp.getvalue()
urllib 超时设置
import socket socket.setdefaulttimeout(5.0)
Python中Cookie的处理:自动处理Cookie,保存为Cookie文件,从文件载入Cookie
Python中的Cookie相关的库
Python中有自带的模块:cookielib
可以处理cookie相关的很多事情。
包括
自动处理cookie:cookie数据是保存在内存中
自动处理cookie:支持Cookie数据存放到文件中
其中有两种格式:
LWP
Mozilla
进一步,再支持:
将cookie存到文件里面
从文件中载入cookie
更多详细格式和解释,请直接看下面的代码。
Python中自动处理Cookie,将Cookie保存到文件,从文件中读取Cookie
Python代码:
#!/usr/bin/python # -*- coding: utf-8 -*- """ Version: 2013-01-15 Author: Crifan Contact: admin (at) crifan.com """ import os; import cookielib; import urllib2; def pythonAutoHandleCookie(): """ Demo how to auto handle cookie in Python cookies in memory cookies in file: save cookie to file LWP format Mozilla format load cookie from file """ print "1. Demo how to auto handle cookie (in memory)"; cookieJarInMemory = cookielib.CookieJar(); opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJarInMemory)); urllib2.install_opener(opener); print "after init, cookieJarInMemory=",cookieJarInMemory; #after init, cookieJarInMemory= #!!! following urllib2 will auto handle cookies demoUrl = "http://www.google.com/"; response = urllib2.urlopen(demoUrl); #here, we already got response cookies print "after urllib2.urlopen, cookieJarInMemory=",cookieJarInMemory; #after urllib2.urlopen, cookieJarInMemory=print "2. Demo how to auto handle cookie in file, LWP format"; cookieFilenameLWP = "localCookiesLWP.txt"; cookieJarFileLWP = cookielib.LWPCookieJar(cookieFilenameLWP); #will create (and save to) new cookie file cookieJarFileLWP.save(); opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJarFileLWP)); urllib2.install_opener(opener); #!!! following urllib2 will auto handle cookies demoUrl = "http://www.google.com/"; response = urllib2.urlopen(demoUrl); #update cookies, save cookies into file cookieJarFileLWP.save(); #for demo, print cookies in file print "LWP cookies:"; print open(cookieFilenameLWP).read(os.path.getsize(cookieFilenameLWP)); # #LWP-Cookies-2.0 # Set-Cookie3: PREF="ID=34c1415b570a93ae:FF=0:NW=1:TM=1358236121:LM=1358236121:S=gEVVojW4x37ht5n-"; path="/"; domain=".google.com"; path_spec; domain_dot; expires="2015-01-15 07:48:41Z"; version=0 # Set-Cookie3: NID="67=JI_uEwUm5GDrQ_vCwAp2z_YGU7MdLm5CLMa4CNLF7RQuTDMzrrk1EjRddGcnpoFbht81LaV9spxZQQInf0mPS6lDrvcRqBBL5NOTmy8SwOzA6HWC3iTIo4-o3fO1Udkv"; path="/"; domain=".google.com.hk"; path_spec; domain_dot; expires="2013-07-17 07:48:41Z"; HttpOnly=None; version=0 # Set-Cookie3: PREF="ID=8f7e4efca89bdb1b:U=f85a4afa4db021aa:FF=2:LD=zh-CN:NW=1:TM=1358236121:LM=1358236121:S=2WR59hDWutdnUJtF"; path="/"; domain=".google.com.hk"; path_spec; domain_dot; expires="2015-01-15 07:48:41Z"; version=0 print "3. Demo how to auto handle cookie in file, Mozilla Format"; cookieFilenameMozilla = "localCookiesMozilla.txt"; cookieJarFileMozilla = cookielib.MozillaCookieJar(cookieFilenameMozilla); #will create (and save to) new cookie file cookieJarFileMozilla.save(); opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJarFileMozilla)); urllib2.install_opener(opener); #!!! following urllib2 will auto handle cookies demoUrl = "http://www.google.com/"; response = urllib2.urlopen(demoUrl); #update cookies, save cookies into file cookieJarFileMozilla.save(); #for demo, print cookies in file print "Mozilla cookies:"; print open(cookieFilenameMozilla).read(os.path.getsize(cookieFilenameMozilla)); # # Netscape HTTP Cookie File # # http://www.netscape.com/newsref/std/cookie_spec.html # # This is a generated file! Do not edit. # .google.com TRUE / FALSE 1421308121 PREF ID=0e05040dd979207c:FF=0:NW=1:TM=1358236121:LM=1358236121:S=jcFid2XgXMIhPUPl # .google.com.hk TRUE / FALSE 1374047321 NID 67=klMI_Z5ZPWDjUYrWSUHIE_kYI77_ziJaL0kWRoUGThagME86LKY7H-MNa2wAMI_GklIwYcD8t82qPinxzLd4GLDbmWT0OVLCXhRj0wQDC57dTNAsTs4lhVR7Yjvj2tfn # .google.com.hk TRUE / FALSE 1421308121 PREF ID=028f8b736db06a9a:U=6ba6d080847c8de6:FF=2:LD=zh-CN:NW=1:TM=1358236121:LM=1358236121:S=_1BcC5v3G0ZojVz8 print "4. read cookies from file"; parseAndSavedCookieFile = "parsedAndSavedCookies.txt" parsedCookieJarFile = cookielib.MozillaCookieJar(parseAndSavedCookieFile); #parsedCookieJarFile = cookielib.MozillaCookieJar(cookieFilenameMozilla); print parsedCookieJarFile; # parsedCookieJarFile.load(cookieFilenameMozilla);
print parsedCookieJarFile; #<_MozillaCookieJar.MozillaCookieJar[
if __name__=="__main__":
pythonAutoHandleCookie();
总结
Python中的cookie的自动处理,比C#好多了。
至少有已有的库可供使用。虽然用法上,需要稍微多多了解后,才知道如何使用的。