-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhn_scraper.rb
More file actions
64 lines (51 loc) · 1.5 KB
/
Copy pathhn_scraper.rb
File metadata and controls
64 lines (51 loc) · 1.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
require 'nokogiri'
require 'open-uri'
require './post.rb'
class Scraper
IDDIGITS = 7
LEVELWIDTH = 40
def initialize(url)
@url = url # url of the hackernews post
# html_file is not necessary
html_file = open(@url)
@doc = Nokogiri::HTML(html_file)
end
def scrape
get_post
get_comments
print_stats
end
private
def get_post
# should these initializing steps be moved to post.rb?
# pass the doc to Post.new?
title = @doc.css('title').text
# url of the article link
url = @doc.css('tr.athing>td.title>a')[0]['href']
# take the text with the class 'score' and split at the space
# take the first part, then convert to integer
points = @doc.css('.score').text.split(" ")[0].to_i
# extracting the last 7 characters of the url only works with a web address
item_id = @url[-IDDIGITS..-1]
@post = Post.new(title, url, points, item_id)
end
def get_comments
# first 'tr.athing' is the title block
comments = @doc.css('tr.athing')
comments.shift
comments.each {|comment|
user_name = comment.css('.comhead > a:first-child').text
time = comment.css('.comhead > a:nth-child(2)').text
# content is bracketed by \n
content = comment.css('.comment').text
level = comment.css('img')[0]['width'].to_i/LEVELWIDTH
comment = Comment.new(user_name, time, content, level)
@post.add_comment(comment)
}
end
def print_stats
puts @post.info
end
end
scraper = Scraper.new(ARGV[0])
scraper.scrape