Question

我正在尝试使用python + selenium + browsermob捕获HAR文件，因为我需要分析加载时间和下载各种页面的内容。但是，由代码生成的文件未命中请求和响应内容字段。在线HAR viewer为生成的har中的所有条目提供了缺失错误：

log.entries [i] .request log.entries [i] .response.status
log.entries [I] .response.statusText

这是我的python脚本：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException,ElementNotVisibleException
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from browsermobproxy import Server
import urlparse
from harpy.har import Har
import json
import sys
import urllib
import urllib2
import re
from pymouse import PyMouse
from pykeyboard import PyKeyboard
import time

#----initialize proxy for packet capture----#
server = Server('C:/browsermob-proxy/bin/browsermob-proxy')
server.start()
proxy = server.create_proxy()
proxy.new_har()

#-----change proxy in chrome --------#
chrome_options = webdriver.ChromeOptions()
url = urlparse.urlparse(proxy.proxy).path
chrome_options.add_argument('--proxy-server=%s' % url) 

driver= webdriver.Chrome(executable_path='C:\Users\user name\Downloads\downloadedsoftware\python\chromedriver_win32\chromedriver.exe',
                         chrome_options=chrome_options)
driver.get("http://www.dcfever.com")

time.sleep(2)
text_file = open("C:\Users\user name\Downloads\webpage.har", "w")
text_file.write(str(proxy.har))
text_file.close() 


time.sleep(5)
proxy.close()
driver.quit()
server.stop()

此外，以下是我获得的部分HAR的输出。由于尺寸限制，我无法粘贴所有内容。

{"log": {"comment": "", "creator": {"comment": "", "version": "2.0", "name": "BrowserMob Proxy"}, "version": "1.2", "entries": [{"comment": "", "serverIPAddress": "203.194.138.68", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.307+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [], "rl": "http://www.dcfever.com/", "qeryString": [], "headers": [], "headersSize": 341, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 17, "send": 2, "ssl": 0, "connect": 2, "dns": 148, "blocked": 12, "wait": 82}, "time": 263, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/html", "comment": "", "size": 21342}, "headers": [], "headersSize": 930, "redirectURL": "", "bodySize": 21342, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.583+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/style/global_26may08.css", "qeryString": [], "headers": [], "headersSize": 438, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 4, "send": 2, "ssl": 0, "connect": 0, "dns": 6, "blocked": 1, "wait": 9}, "time": 22, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 4237}, "headers": [], "headersSize": 627, "redirectURL": "", "bodySize": 4237, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.68", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.587+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://www.dcfever.com/ads/geoads_mpsb.php", "qeryString": [], "headers": [], "headersSize": 414, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 0, "send": 0, "ssl": 0, "connect": 2, "dns": 0, "blocked": 0, "wait": 10}, "time": 12, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/javascript", "comment": "", "size": 250}, "headers": [], "headersSize": 517, "redirectURL": "", "bodySize": 250, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.587+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/style/style_01sept08_5.css", "qeryString": [], "headers": [], "headersSize": 440, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 26, "send": 0, "ssl": 0, "connect": 1, "dns": 10, "blocked": 0, "wait": 6}, "time": 43, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 68955}, "headers": [], "headersSize": 627, "redirectURL": "", "bodySize": 68955, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.595+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/style/index.css?date=12092013", "qeryString": [{"name": "date", "vale": "12092013"}], "headers": [], "headersSize": 443, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 7, "send": 0, "ssl": 0, "connect": 1, "dns": 0, "blocked": 1, "wait": 35}, "time": 44, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 12040}, "headers": [], "headersSize": 621, "redirectURL": "", "bodySize": 12040, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.596+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/style/header_style_mega.css?date=06092013", "qeryString": [{"name": "date", "vale": "06092013"}], "headers": [], "headersSize": 455, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 51, "send": 0, "ssl": 0, "connect": 1, "dns": 4, "blocked": 0, "wait": 30}, "time": 86, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 18872}, "headers": [], "headersSize": 618, "redirectURL": "", "bodySize": 18872, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.596+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/style/global_mega_narrow.css?date=06092013c", "qeryString": [{"name": "date", "vale": "06092013c"}], "headers": [], "headersSize": 457, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 5, "send": 0, "ssl": 0, "connect": 1, "dns": 0, "blocked": 1, "wait": 33}, "time": 40, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 6152}, "headers": [], "headersSize": 617, "redirectURL": "", "bodySize": 6152, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "173.194.72.95", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.593+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [], "rl": "http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js", "qeryString": [], "headers": [], "headersSize": 348, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 69, "send": 0, "ssl": 0, "connect": 1, "dns": 4, "blocked": 0, "wait": 68}, "time": 142, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/javascript; charset=UTF-8", "comment": "", "size": 32819}, "headers": [], "headersSize": 682, "redirectURL": "", "bodySize": 32819, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "173.194.72.95", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.591+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [], "rl": "http://ajax.googleapis.com/ajax/libs/jqueryi/1.10.3/jquery-i.min.js", "qeryString": [], "headers": [], "headersSize": 354, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 108, "send": 0, "ssl": 0, "connect": 2, "dns": 8, "blocked": 0, "wait": 71}, "time": 189, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/javascript; charset=UTF-8", "comment": "", "size": 60666}, "headers": [], "headersSize": 682, "redirectURL": "", "bodySize": 60666, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "173.194.72.95", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.595+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [], "rl": "http://ajax.googleapis.com/ajax/libs/swfobject/2.2/swfobject.js", "qeryString": [], "headers": [], "headersSize": 348, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 2, "send": 0, "ssl": 0, "connect": 2, "dns": 3, "blocked": 0, "wait": 2}, "time": 9, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/javascript; charset=UTF-8", "comment": "", "size": 3944}, "headers": [], "headersSize": 679, "redirectURL": "", "bodySize": 3944, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.594+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/style/ser_review_style_2013.css", "qeryString": [], "headers": [], "headersSize": 446, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 4, "send": 0, "ssl": 0, "connect": 1, "dns": 1, "blocked": 0, "wait": 11}, "time": 17, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 13633}, "headers": [], "headersSize": 630, "redirectURL": "", "bodySize": 13633, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.68", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.597+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://www.dcfever.com/js/jquery.cookie.js", "qeryString": [], "headers": [], "headersSize": 414, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 4, "send": 0, "ssl": 0, "connect": 4, "dns": 3, "blocked": 1, "wait": 14}, "time": 26, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "application/x-javascript", "comment": "", "size": 4480}, "headers": [], "headersSize": 619, "redirectURL": "", "bodySize": 4480, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.82", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.595+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [], "rl": "http://adx.adxglobal.com/ads/www/delivery/spcjs.php?id=1&charset=UTF-8", "qeryString": [{"name": "charset", "vale": "UTF-8"}, {"name": "id", "vale": "1"}], "headers": [], "headersSize": 353, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 4, "send": 0, "ssl": 0, "connect": 0, "dns": 93, "blocked": 0, "wait": 96}, "time": 193, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "application/x-javascript", "comment": "", "size": 733}, "headers": [], "headersSize": 624, "redirectURL": "", "bodySize": 733, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.68", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.596+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://www.dcfever.com/ads/geoads.php", "qeryString": [], "headers": [], "headersSize": 409, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 0, "send": 0, "ssl": 0, "connect": 2, "dns": 0, "blocked": 0, "wait": 11}, "time": 13, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/javascript", "comment": "", "size": 26}, "headers": [], "headersSize": 517, "redirectURL": "", "bodySize": 26, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.68", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.596+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://www.dcfever.com/images/dcfever_logo_png.png", "qeryString": [], "headers": [], "headersSize": 439, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 0, "send": 0, "ssl": 0, "connect": 0, "dns": 0, "blocked": 0, "wait": 17}, "time": 17, "response": {"stats": 301, "comment": "", "cookies": [], "statsText": "Moved Permanently", "content": {"mimeType": "text/html", "comment": "", "size": 184}, "headers": [], "headersSize": 541, "redirectURL": "", "bodySize": 184, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.606+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/reviews/jquery-i.css", "qeryString": [], "headers": [], "headersSize": 435, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 25, "send": 0, "ssl": 0, "connect": 1, "dns": 0, "blocked": 0, "wait": 8}, "time": 34, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/css", "comment": "", "size": 32046}, "headers": [], "headersSize": 627, "redirectURL": "", "bodySize": 32046, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.655+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/js/jquery.dropdownPlain.js", "qeryString": [], "headers": [], "headersSize": 425, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 1, "send": 0, "ssl": 0, "connect": 1, "dns": 0, "blocked": 0, "wait": 10}, "time": 12, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "application/x-javascript", "comment": "", "size": 374}, "headers": [], "headersSize": 632, "redirectURL": "", "bodySize": 374, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.675+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/js/hoverIntent.js", "qeryString": [], "headers": [], "headersSize": 416, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 3, "send": 0, "ssl": 0, "connect": 2, "dns": 0, "blocked": 1, "wait": 7}, "time": 13, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "application/x-javascript", "comment": "", "size": 5054}, "headers": [], "headersSize": 633, "redirectURL": "", "bodySize": 5054, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "203.194.138.69", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.675+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [{"comment": "", "name": "dcff_", "vale": "0"}, {"comment": "", "name": "dcff_sid", "vale": "3ae38cbd15e0f6a15d788783876251fb"}, {"comment": "", "name": "ssid", "vale": "20150320165750.Sz5V7260"}], "rl": "http://cdn01.dcfever.com/images/dcfever_logo_png.png", "qeryString": [], "headers": [], "headersSize": 443, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 2, "send": 0, "ssl": 0, "connect": 2, "dns": 0, "blocked": 1, "wait": 7}, "time": 12, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "image/png", "comment": "", "size": 14300}, "headers": [], "headersSize": 632, "redirectURL": "", "bodySize": 14300, "httpVersion": "HTTP/1.0"}}, {"comment": "", "serverIPAddress": "173.194.127.249", "pageref": "Page 0", "startedDateTime": "2015-03-20T16:57:42.723+08:00", "cache": {}, "reqest": {"comment": "", "cookies": [], "rl": "http://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js", "qeryString": [], "headers": [], "headersSize": 356, "bodySize": 0, "method": "GET", "httpVersion": "HTTP/1.1"}, "timings": {"comment": "", "receive": 3, "send": 0, "ssl": 0, "connect": 1, "dns": 63, "blocked": 0, "wait": 8}, "time": 75, "response": {"stats": 200, "comment": "", "cookies": [], "statsText": "OK", "content": {"mimeType": "text/javascript; charset=UTF-8", "comment": "", "size": 12645}, "headers": [], "headersSize": 742, "redirectURL": "", "bodySize": 12645, "httpVersion": "HTTP/1.0"........}}}}}}]}}}

请建议我该怎么做。

Answer 1

请尝试此代码。这将给你HAR。请根据需要更改Chrome或其他浏览器。

from browsermobproxy import Server
from selenium import webdriver
import os, json, re, urllib2


def Generate_HAR_from_url(browsermob_location, url, port_browsermob=9090):
    port_browsermob = int(port_browsermob)
    browsermob_location = os.path.abspath(browsermob_location)
    server = Server(browsermob_location,options={'port':port_browsermob})
    server.start()
    proxy = server.create_proxy()
    profile  = webdriver.FirefoxProfile()
    profile.set_proxy(proxy.selenium_proxy())
    driver = webdriver.Firefox(firefox_profile=profile)
    proxy.new_har("generated_har")
    driver.get(url)
    result_har = proxy.har # returns a HAR JSON blob
    proxy.close()
    server.stop()
    driver.quit()
    return result_har

Harfile Python + Selenium + browsermob中的问题

1 个答案: