INSERTs (in progress)

urq.js

@@ -19,6 +19,7 @@ class Urq {
       port:     '3306',
       query:    {pool: true}
     };
+    this.forum_id = 2;// your first category
   };
 
   async scrape() {
@@ -109,8 +110,8 @@ class Urq {
         for (let i = 0; i < msg.args().length; ++i)
           console.log(`${i}: ${msg.args()[i]}`);
       });
-      const [rows, fields] = await this.db.execute(`
-        SELECT \`url\` FROM \`${config.DB_PREFIX}bordathreads\`
+      let [rows, fields] = await this.db.execute(`
+        SELECT * FROM \`${config.DB_PREFIX}bordathreads\`
       `);
       for (let i = 0; i <= rows.length;i++) {
         await page.goto(rows[i].url, {
@@ -134,14 +135,13 @@ class Urq {
               if(rows[i].getAttribute("class") !== null) {
                 rowclasses = rows[i].getAttribute("class").split(" ");
               }
-              console.log(rowclasses);
               if (
                 rowclasses.length > 0 &&
                 intersect(msgclasses, rowclasses) === []
               ) {
                 continue;
               }
-              let date = undefined;
+              let date = null;
               let author = rows[i].querySelector(".font3 a b");
               if (author) {
                 author = author.innerHTML;
@@ -162,12 +162,13 @@ class Urq {
               if (avatar) {
                 avatar = avatar.getAttribute("src");
                 if (avatar === "http://forum24.ru/gif/img/p.gif") {
-                  avatar = undefined;
+                  avatar = null;
                 }
               }
               if (author && content) {
                 retval.push({
                   "author": author,
+                  "title": "",
                   "content": content,
                   "avatar": avatar,
                   "date": date
@@ -181,8 +182,61 @@ class Urq {
           }
         });
         console.log(messages);
-        return;
+
+        let user_id = -1;
+        let topic_id = -1;
+        let temp;
+        let userrows = await this.db.execute(`
+          SELECT \`user_id\` FROM \`${config.DB_PREFIX}users\`
+          WHERE username = "${messages[0].author}";
+        `);
+        if (userrows.length > 0) {
+          user_id = userrows[0].user_id;
+        } else {
+          temp = await this.db.execute(`
+            INSERT INTO \`${config.DB_PREFIX}users\`
+            (
+              \`username\`, \`username_clean\`, \`user_permissions\`,
+              \`user_sig\`
+            )
+            VALUES(?, ?, ?, ?);`,[
+              messages[0].author,
+              messages[0].author.toLowerCase(),
+              "",
+              ""
+            ]);
+          user_id = temp.insertId;
+        }
+        temp = await this.db.execute(`
+          INSERT INTO \`${config.DB_PREFIX}topics\`
+          (\`forum_id\`, \`topic_poster\`, \`topic_time\`, \`topic_title\`)
+          VALUES(?, ?, ?, ?);`,[
+            this.forum_id,
+            user_id,
+            messages[0].date,
+            messages[0].title,
+            rows[i].name
+          ]);
+        topic_id = temp.insertId;
+        for (let j = 0; j <= messages.length;j++) {
+          await this.db.execute(`
+            INSERT INTO \`${config.DB_PREFIX}posts\`
+            (
+              \`topic_id\`, \`forum_id\`, \`poster_id\`,
+              \`post_time\`, \`post_subject\`, \`post_text\`
+            )
+            VALUES(?, ?, ?, ?);`,[
+              topic_id,
+              this.forum_id,
+              user_id,
+              messages[j].date,
+              messages[j].title,
+              messages[j].content
+            ]);
+        }
       }
+      // DEBUG - only first page
+      return;
     } catch(e) {
       console.log(e);
     }