我正在从YouTube频道和视频中检索元数据。
一切都很顺利,但目前我正在努力将所有信息放入所需的数据框中。以下是我使用的代码,来自此Github https://gist.github.com/andkamau/0d4e312c97f41a975440a05fd76b1d29。
一切都很顺利,但目前我正在努力将所有信息放入所需的数据框中。以下是我使用的代码,来自此Github https://gist.github.com/andkamau/0d4e312c97f41a975440a05fd76b1d29。
import urllib.request
import json
from bs4 import BeautifulSoup
from collections import namedtuple
import pafy
from pandas import *
import pandas as pd
df = pd.DataFrame()
Video = namedtuple("Video", "video_id title duration views thumbnail Description")
def parse_video_div(div):
video_id = div.get("data-context-item-id", "")
title = div.find("a", "yt-uix-tile-link").text
duration = div.find("span", "video-time").contents[0].text
views = str(div.find("ul", "yt-lockup-meta-info").contents[0].text.rstrip(" views").replace(",", ""))
img = div.find("img")
videoDescription = pafy.new("https://www.youtube.com/watch?v="+video_id)
thumbnail = "http:" + img.get("src", "") if img else ""
Description = videoDescription.description
l = Video(video_id, title, duration, views, thumbnail, Description)
# storing in the dataframe
df = pd.DataFrame(list(Video(video_id, title, duration, views, thumbnail, Description)))
return Video(video_id, title, duration, views, thumbnail, Description)
def parse_videos_page(page):
video_divs = page.find_all("div", "yt-lockup-video")
return [parse_video_div(div) for div in video_divs]
def find_load_more_url(page):
for button in page.find_all("button"):
url = button.get("data-uix-load-more-href")
if url:
return "http://www.youtube.com" + url
def download_page(url):
print("Downloading {0}".format(url))
return urllib.request.urlopen(url).read()
def get_videos(username):
page_url = "http://www.youtube.com/channel/{0}/videos".format(username)
page = BeautifulSoup(download_page(page_url))
videos = parse_videos_page(page)
page_url = find_load_more_url(page)
while page_url:
json_data = json.loads(str(download_page(page_url).decode("utf-8")))
page = BeautifulSoup(json_data.get("content_html", ""))
videos.extend(parse_videos_page(page))
page_url = find_load_more_url(BeautifulSoup(json_data.get("load_more_widget_html", "")))
return videos
if __name__ == "__main__":
videos = get_videos("UC-M9eLhclbe16sDaxLzc0ng")
for video in videos:
print(video)
print("{0} videos".format(len(videos)))
函数parse_video_div(div)
包含所有信息和我的dataframe
。但不幸的是,dataframe
没有返回任何内容。也许我需要以某种方式循环使用namedtuple
。
有什么线索可以帮助我实现dataframe
看到我的数据吗?
parse_video_div
函数中没有返回你的数据框。所以你将视频转换为变量df
中的数据框,然后却没有对它进行任何操作。 - grofte