Skip to content

Commit 75effef

Browse files
author
Karen Z
committed
Add scraping example
1 parent 41fb214 commit 75effef

File tree

2 files changed

+250
-2
lines changed

2 files changed

+250
-2
lines changed

‎.ipynb_checkpoints/gfg_0_get_top_comments-checkpoint.ipynb

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
"metadata": {},
1919
"outputs": [],
2020
"source": [
21-
"client_id = 'REDACTED'\n",
22-
"client_secret = 'REDACTED'\n",
21+
"client_id = 'REDACTED' # use your client ID\n",
22+
"client_secret = 'REDACTED' # use your client secret\n",
2323
"user_agent = 'Private project reading public info only'"
2424
]
2525
},

‎gfg_0_get_top_comments.ipynb

+248
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"import praw\n",
10+
"import pandas as pd\n",
11+
"import datetime as dt\n",
12+
"import pprint"
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": 2,
18+
"metadata": {},
19+
"outputs": [],
20+
"source": [
21+
"client_id = 'REDACTED' # use your client ID\n",
22+
"client_secret = 'REDACTED' # use your client secret\n",
23+
"user_agent = 'Private project reading public info only'"
24+
]
25+
},
26+
{
27+
"cell_type": "code",
28+
"execution_count": 3,
29+
"metadata": {},
30+
"outputs": [],
31+
"source": [
32+
"reddit = praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)"
33+
]
34+
},
35+
{
36+
"cell_type": "code",
37+
"execution_count": 4,
38+
"metadata": {},
39+
"outputs": [],
40+
"source": [
41+
"gild_dict = { \"id\":[],\n",
42+
" \"parent_id\":[],\n",
43+
" \"link_id\":[],\n",
44+
" \"gilded\":[],\n",
45+
" \"author\":[],\n",
46+
" \"created\":[],\n",
47+
" \"is_submitter\":[],\n",
48+
" \"permalink\":[],\n",
49+
" \"score\":[],\n",
50+
" \"body\":[],\n",
51+
" \"question\":[] }"
52+
]
53+
},
54+
{
55+
"cell_type": "code",
56+
"execution_count": 5,
57+
"metadata": {},
58+
"outputs": [],
59+
"source": [
60+
"askreddit = reddit.subreddit('askreddit')"
61+
]
62+
},
63+
{
64+
"cell_type": "code",
65+
"execution_count": 7,
66+
"metadata": {},
67+
"outputs": [
68+
{
69+
"name": "stdout",
70+
"output_type": "stream",
71+
"text": [
72+
"Submission no. 51\n"
73+
]
74+
},
75+
{
76+
"ename": "TooLarge",
77+
"evalue": "received 413 HTTP response",
78+
"output_type": "error",
79+
"traceback": [
80+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
81+
"\u001b[0;31mTooLarge\u001b[0m Traceback (most recent call last)",
82+
"\u001b[0;32m<ipython-input-7-839f5542120d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Submission no. \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcounter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mquestion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubmission\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0msubmission\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace_more\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcomment\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubmission\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mcomment_counter\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
83+
"\u001b[0;32m//anaconda3/lib/python3.7/site-packages/praw/models/comment_forest.py\u001b[0m in \u001b[0;36mreplace_more\u001b[0;34m(self, limit, threshold)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m \u001b[0mnew_comments\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 166\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mremaining\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0mremaining\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
84+
"\u001b[0;32m//anaconda3/lib/python3.7/site-packages/praw/models/reddit/more.py\u001b[0m in \u001b[0;36mcomments\u001b[0;34m(self, update)\u001b[0m\n\u001b[1;32m 74\u001b[0m }\n\u001b[1;32m 75\u001b[0m self._comments = self._reddit.post(\n\u001b[0;32m---> 76\u001b[0;31m \u001b[0mAPI_PATH\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"morechildren\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 77\u001b[0m )\n\u001b[1;32m 78\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
85+
"\u001b[0;32m//anaconda3/lib/python3.7/site-packages/praw/reddit.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(self, path, data, files, params)\u001b[0m\n\u001b[1;32m 525\u001b[0m \"\"\"\n\u001b[1;32m 526\u001b[0m data = self.request(\n\u001b[0;32m--> 527\u001b[0;31m \u001b[0;34m\"POST\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 528\u001b[0m )\n\u001b[1;32m 529\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_objector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobjectify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
86+
"\u001b[0;32m//anaconda3/lib/python3.7/site-packages/praw/reddit.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, path, params, data, files)\u001b[0m\n\u001b[1;32m 579\u001b[0m \"\"\"\n\u001b[1;32m 580\u001b[0m return self._core.request(\n\u001b[0;32m--> 581\u001b[0;31m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 582\u001b[0m )\n\u001b[1;32m 583\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
87+
"\u001b[0;32m//anaconda3/lib/python3.7/site-packages/prawcore/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, path, data, files, json, params)\u001b[0m\n\u001b[1;32m 183\u001b[0m return self._request_with_retries(\n\u001b[1;32m 184\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m params=params, url=url)\n\u001b[0m\u001b[1;32m 186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
88+
"\u001b[0;32m//anaconda3/lib/python3.7/site-packages/prawcore/sessions.py\u001b[0m in \u001b[0;36m_request_with_retries\u001b[0;34m(self, data, files, json, method, params, url, retries)\u001b[0m\n\u001b[1;32m 128\u001b[0m retries, saved_exception, url)\n\u001b[1;32m 129\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSTATUS_EXCEPTIONS\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSTATUS_EXCEPTIONS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 131\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mcodes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'no_content'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
89+
"\u001b[0;31mTooLarge\u001b[0m: received 413 HTTP response"
90+
]
91+
}
92+
],
93+
"source": [
94+
"counter = 0\n",
95+
"for submission in askreddit.top(limit=100):\n",
96+
" counter += 1\n",
97+
" if counter <= 50:\n",
98+
" continue\n",
99+
" comment_counter = 0\n",
100+
" print(\"Submission no. \", counter)\n",
101+
" question = submission.title\n",
102+
" submission.comments.replace_more(limit=None)\n",
103+
" for comment in list(submission.comments):\n",
104+
" comment_counter += 1\n",
105+
" gild_dict[\"id\"].append(comment.id) if comment.id else gild_dict[\"id\"].append(\"None\")\n",
106+
" gild_dict[\"parent_id\"].append(comment.parent_id) if comment.parent_id else gild_dict[\"parent_id\"].append(\"None\")\n",
107+
" gild_dict[\"link_id\"].append(comment.link_id) if comment.link_id else gild_dict[\"link_id\"].append(\"None\")\n",
108+
" gild_dict[\"gilded\"].append(comment.gilded) \n",
109+
" gild_dict[\"author\"].append(comment.author) if comment.author else gild_dict[\"author\"].append(\"None\")\n",
110+
" gild_dict[\"created\"].append(comment.created_utc) if comment.created_utc else gild_dict[\"created_utc\"].append(\"None\")\n",
111+
" gild_dict[\"is_submitter\"].append(comment.is_submitter) \n",
112+
" gild_dict[\"permalink\"].append(comment.permalink) if comment.permalink else gild_dict[\"permalink\"].append(\"None\")\n",
113+
" gild_dict[\"score\"].append(comment.score)\n",
114+
" gild_dict[\"body\"].append(comment.body) if comment.body else gild_dict[\"body\"].append(\"None\")\n",
115+
" gild_dict[\"question\"].append(question)\n",
116+
" if comment_counter >= 100:\n",
117+
" print(\"Got 100!\")\n",
118+
" break"
119+
]
120+
},
121+
{
122+
"cell_type": "code",
123+
"execution_count": 7,
124+
"metadata": {},
125+
"outputs": [
126+
{
127+
"data": {
128+
"text/plain": [
129+
"11"
130+
]
131+
},
132+
"execution_count": 7,
133+
"metadata": {},
134+
"output_type": "execute_result"
135+
}
136+
],
137+
"source": [
138+
"len(gild_dict)"
139+
]
140+
},
141+
{
142+
"cell_type": "code",
143+
"execution_count": 8,
144+
"metadata": {},
145+
"outputs": [],
146+
"source": [
147+
"df_tops = pd.DataFrame(gild_dict)"
148+
]
149+
},
150+
{
151+
"cell_type": "code",
152+
"execution_count": 9,
153+
"metadata": {},
154+
"outputs": [
155+
{
156+
"data": {
157+
"text/plain": [
158+
"3500"
159+
]
160+
},
161+
"execution_count": 9,
162+
"metadata": {},
163+
"output_type": "execute_result"
164+
}
165+
],
166+
"source": [
167+
"len(df_tops)"
168+
]
169+
},
170+
{
171+
"cell_type": "code",
172+
"execution_count": 11,
173+
"metadata": {},
174+
"outputs": [
175+
{
176+
"data": {
177+
"text/plain": [
178+
"279"
179+
]
180+
},
181+
"execution_count": 11,
182+
"metadata": {},
183+
"output_type": "execute_result"
184+
}
185+
],
186+
"source": [
187+
"(df_tops['gilded'] >= 1).sum()"
188+
]
189+
},
190+
{
191+
"cell_type": "code",
192+
"execution_count": 12,
193+
"metadata": {},
194+
"outputs": [
195+
{
196+
"data": {
197+
"text/plain": [
198+
"0.07971428571428571"
199+
]
200+
},
201+
"execution_count": 12,
202+
"metadata": {},
203+
"output_type": "execute_result"
204+
}
205+
],
206+
"source": [
207+
"279/3500"
208+
]
209+
},
210+
{
211+
"cell_type": "code",
212+
"execution_count": 10,
213+
"metadata": {},
214+
"outputs": [],
215+
"source": [
216+
"df_tops.to_csv('data/askreddit_tops_16_51.csv', index=False)"
217+
]
218+
},
219+
{
220+
"cell_type": "code",
221+
"execution_count": null,
222+
"metadata": {},
223+
"outputs": [],
224+
"source": []
225+
}
226+
],
227+
"metadata": {
228+
"kernelspec": {
229+
"display_name": "Python 3",
230+
"language": "python",
231+
"name": "python3"
232+
},
233+
"language_info": {
234+
"codemirror_mode": {
235+
"name": "ipython",
236+
"version": 3
237+
},
238+
"file_extension": ".py",
239+
"mimetype": "text/x-python",
240+
"name": "python",
241+
"nbconvert_exporter": "python",
242+
"pygments_lexer": "ipython3",
243+
"version": "3.7.3"
244+
}
245+
},
246+
"nbformat": 4,
247+
"nbformat_minor": 2
248+
}

0 commit comments

Comments
 (0)