oreolek/urqforum-parser
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Scrape all pages

Browse source
This commit is contained in:
Alexander Yakovlev 2018-04-10 17:36:08 +07:00
parent e731cc6ee7
commit b7afda5385
1 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
urq.js
View file

@ -50,8 +50,7 @@ class Urq {
for (let offset = 0; offset <= 400; offset += 20) {
console.log("Offset "+offset);
await page.goto("http://urq.borda.ru?0-0-"+offset, {
"waitUntil" : "load",
"timeout": 60000
"waitUntil" : "domcontentloaded"
});
let pagethreads = await page.evaluate(() => {
@ -60,7 +59,7 @@ class Urq {
let href = [];
for (let i = 0; i < links.length; i++) {
href.push({
"url": links[i].getAttribute("href"),
"url": "http://urq.borda.ru"+links[i].getAttribute("href"),
"name": links[i].innerHTML,
});
}
@ -73,10 +72,10 @@ class Urq {
for (let i = 0; i < pagethreads.length; i++) {
threads.push(pagethreads[i]);
}
process.exit();
}
console.log(threads);
for (let i = 0; i < threads.length; i++) {
try {
let [rows, fields] = await this.db.execute(`