python 模拟浏览器 new

https://www.sodocs.net/doc/7918306844.html,/maseng/p/3578553.html
想用python模拟浏览器访问web的方法测试些东西，有哪几种方法呢？

一类：单纯的访问web，不解析其js，css等。

1. urllib2

#-*- coding:utf-8 -*

import urllib2



def Furllib2(ip,port,url,timeout):

proxydict = {}

proxydict['http'] = "http://%s:%s"%(ip,port)

print proxydict

proxy_handler = urllib2.ProxyHandler(proxydict)

opener = urllib2.build_opener(proxy_handler)

opener.addheaders = [('User-agent', 'Mozilla/5.0')]

urllib2.install_opener(opener)

try:

response = urllib2.urlopen(url,timeout=timeout)

print response.geturl()

print response.getcode()

print https://www.sodocs.net/doc/7918306844.html,()

print response.read()

return True

except:

print 'some errors occored' + '-'*50

return 0



def main():

proxyip = '14.18.16.69'

proxyport = '80'

proxy = 'http://2.181.1.127:80'

url = 'https://www.sodocs.net/doc/7918306844.html,/'

timeout = 4

print Furllib2(proxyip,proxyport,url,timeout)





if __name__ == "__main__":

main()


2. mechanize(与网站的自动化交互)

https://www.sodocs.net/doc/7918306844.html,/mechanize/doc.html


def Fmechanize(url):

cookies = mechanize.CookieJar()

opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookies))

try:

r = opener.open(url) # GET

# r = opener.open("https://www.sodocs.net/doc/7918306844.html,/", data) # POST

print r.geturl()

print https://www.sodocs.net/doc/7918306844.html,()

return True



except:

return 0


二类：模拟浏览器，使用firefox等的浏览器引擎，支持js，css等。

1. selenium 的firefox或者chrome等驱动，但是由于要打开一个浏览器，所以会比较慢（浏览器驱动可以到selenium官网上下载，也可以到firefox插件出搜索）



def Fselenium_firefox(ip,port,url,timeout):

try:



profile = webdriver.FirefoxProfile()

profile.set_preference('network.proxy.type', 1)

profile.set_preference('network.proxy.http',ip)

profile.set_preference('network.proxy.http_port', port)

profile.update_preferences()

driver = webdriver.Firefox(profile,timeout = timeout)

except Exception:

print traceback.print_exc()

return 0

pass

try:



driver.get(url)

time.sleep(5)

cookies= driver.get_cookies()

print cookies

# driver.get()



driver.quit()

return 1





except Exception:

traceback.print_exc()

# print 'not have Union allianceid'

driver.quit()

return 0


2. selenium :headless test使用selenium+ phantomjs驱动，无需打开浏览器，但是支持js的模拟浏览器动作，也就说说和你手工打开是

没有区别的。

https://www.sodocs.net/doc/7918306844.html,/git/docs/api/py/api.html


def Fselenium_phantomjs(ip,port,url,timeout):

try:

proxyip = '%s%s%s%s'%('--proxy=',ip,':',port)

proxyport = '--proxy-type=http'

service_args = []

service_args.append(proxyip)

service_args.append(proxyport)

print service_args



driver = webdriver.PhantomJS(service_args = service_args)
#driver = webdriver.PhantomJS("/root/phantomjs-1.9.7-linux-x86_64/bin/phantomjs",service_args = service_args)制定phantomjs的位置

driver.set_page_load_timeout(timeout)

driver.get(url)

time.sleep(4)

except Exception:

traceback.print_exc()



try:

geturl = driver.current_url

print driver.current_url

return True

except Exception:

traceback.print_exc()

geturl = None

return 0


3. qt，网上戗来的代码

https://www.sodocs.net/doc/7918306844.html,/wiki/PySide#PySide.QtWebKit.PySide.QtWebKit.QWebView.url



from PyQt4 import QtCore, QtGui, QtWebKit, QtNetwork



class cookieJar(QtNetwork.QNetworkCookieJar):

def __init__(self, cookiesKey, parent=None):

super(cookieJar, self).__init__(parent)



self.mainWindow = parent

self.cookiesKey = cookiesKey

cookiesValue = self.mainWindow.settings.value(self.cookiesKey)



if cookiesValue:

cookiesList = QtNetwork.QNetworkCookie.parseCookies(cookiesValue)

self.setAllCookies(cookiesList)



# def setCookiesFromUrl (self, cookieList, url):

# cookiesValue = self.mainWindow.settings.value(self.cookiesKey)

# cookiesArray = cookiesValue if cookiesValue else QtCore.QByteArray()



# for cookie in cookieList:

# cookiesArray.append(cookie.toRawForm() + "\n")



#self.mainWindow.settings.setValue(self.cookiesKey, cookiesArray)



#return super(cookieJar, self).setCookiesFromUrl(cookieList, url)

def deleteCookie(self,cookieList):

cookie = []

self.mainWindow.settings.value(cookie)

class webView(QtWebKit.QWebView):

def __init__(self, cookiesKey, url, parent=None):

super(webView, self).__init__(parent)



self.cookieJar = cookieJar(cookiesKey, parent)



self.page().networkAccessManager().setCookieJar(self.cookieJar)



class myWindow(QtGui.QMainWindow):

def __init__(self, parent=None):

super(myWindow, self).__init__(parent)



self.cookiesKey = "cookies"



self.centralwidget = QtGui.QWidget(self)



self.tabWidget = QtGui.QTabWidget(self.centralwidget)

self.tabWidget.setTabsClosable(True)



self.verticalLayout = QtGui.QVBoxLayout(self.centralwidget)

self.verticalLayout.addWidget(self.tabW

idget)



self.actionTabAdd = QtGui.QAction(self)

self.actionTabAdd.setText("Add Tab")

self.actionTabAdd.triggered.connect(self.on_actionTabAdd_triggered)



self.lineEdit = QtGui.QLineEdit(self)

self.lineEdit.setText("https://www.sodocs.net/doc/7918306844.html,")



self.toolBar = QtGui.QToolBar(self)

self.toolBar.addAction(self.actionTabAdd)

self.toolBar.addWidget(self.lineEdit)



self.addToolBar(QtCore.Qt.ToolBarArea(QtCore.Qt.TopToolBarArea), self.toolBar)

self.setCentralWidget(self.tabWidget)



self.settings = QtCore.QSettings()



@QtCore.pyqtSlot()

def on_actionShowCookies_triggered(self):

webView = self.tabWidget.currentWidget()

listCookies = webView.page().networkAccessManager().cookieJar().allCookies()



for cookie in listCookies:

print cookie.toRawForm()



@QtCore.pyqtSlot()

def on_actionTabAdd_triggered(self):

url = self.lineEdit.text()

self.addNewTab(url if url else 'about:blank')



def addNewTab(self, url):

tabName = u"Tab {0}".format(str(self.tabWidget.count()))



tabWidget= webView(self.cookiesKey, url, self)

tabWidget.loadFinished.connect(self.on_tabWidget_loadFinished)

tabWidget.load(QtCore.QUrl(url))



tabIndex = self.tabWidget.addTab(tabWidget, tabName)



self.tabWidget.setCurrentIndex(tabIndex)



@QtCore.pyqtSlot()

def on_tabWidget_loadFinished(self):

cookies2 = self.settings.value(self.cookiesKey)





if __name__ == "__main__":

import sys



app = QtGui.QApplication(sys.argv)

app.setApplicationName('myWindow')



main = myWindow()

main.resize(666, 333)

main.show()



sys.exit(app.exec_())




4. qt-headless

https://www.sodocs.net/doc/7918306844.html,/wiki/PySide#PySide.QtWebKit.PySide.QtWebKit.QWebView.url




import sys

from PyQt4.QtGui import *

from PyQt4.QtCore import *

from PyQt4.QtWebKit import *



class Render(QWebPage):

def __init__(self, url):

self.app = QApplication(sys.argv)

QWebPage.__init__(self)

self.loadFinished.connect(self._loadFinished)

self.mainFrame().load(QUrl(url))

self.app.exec_()



def _loadFinished(self, result):

self.frame = self.mainFrame()

self.app.quit()



url = 'https://www.sodocs.net/doc/7918306844.html,'

r = Render(url)

html = r.frame.toHtml()

print html


5. splinter :打开浏览器，模拟操作，python的

https://www.sodocs.net/doc/7918306844.html,/docs/tutorial.html






>>> from splinter import Browser

>>> browser = Browser()

>>> url = "https://www.sodocs.net/doc/7918306844.html,"

>>> browser.visit(url)








具体用哪个要看你有什么具体的需求了