Jonth
New member
My app starts with a simple html form. the inputs are PIN# and Date Of Birth.
My express server runs on the same port 3000, when the user submits their data, puppeteer starts and logs into a specific webpage. Then I scrape the image on that webpage. Google Api takes the text from that image and saves it in an array. I then post that array string to src/results.html. But as soon as the user hits submit, they are redirected to /resuts route right immediately and the page says cannot post the data. but when I see in the console (roughly a minute later) that the post was successful, I refresh the page and I get the array of text I wanted to see.
How can I await for the data to finish being posted to the route before the page loads the data? Im using react for client side. below is my server side code. client side is just a basic react page login and a static /results page meant for the data.
My express server runs on the same port 3000, when the user submits their data, puppeteer starts and logs into a specific webpage. Then I scrape the image on that webpage. Google Api takes the text from that image and saves it in an array. I then post that array string to src/results.html. But as soon as the user hits submit, they are redirected to /resuts route right immediately and the page says cannot post the data. but when I see in the console (roughly a minute later) that the post was successful, I refresh the page and I get the array of text I wanted to see.
How can I await for the data to finish being posted to the route before the page loads the data? Im using react for client side. below is my server side code. client side is just a basic react page login and a static /results page meant for the data.
Code:
const puppeteer = require("puppeteer");
const express = require("express");
const app = express();
const morgan = require("morgan");
const fs = require("fs");
const cors = require("cors");
const request = require("request-promise-native").defaults({ Jar: true });
const poll = require("promise-poller").default;
app.use(morgan("combined"));
const port = 3000;
// Imports the Google Cloud client library
const vision = require("@google-cloud/vision");
require("dotenv").config();
app.use(cors());
const textArray = [];
const App = (pinNum, dateOfB) => {
const config = {
sitekey: process.env.SITEKEY,
pageurl: process.env.PAGEURL,
apiKey: process.env.APIKEY,
apiSubmitUrl: "http://2captcha.com/in.php",
apiRetrieveUrl: "http://2captcha.com/res.php",
};
const chromeOptions = {
executablePath: "/Program Files/Google/Chrome/Application/chrome.exe",
headless: true,
slowMo: 60,
defaultViewport: null,
};
async function main() {
const browser = await puppeteer.launch(chromeOptions);
const page = await browser.newPage();
console.log(`Navigating to ${config.pageurl}`);
await page.goto(config.pageurl);
try {
const requestId = await initiateCaptchaRequest(config.apiKey);
// const pin = getPIN();
console.log(`Typing PIN ${pinNum}`);
await page.type("#PIN", pinNum);
// const dob = getDOB();
console.log(`Typing DOB ${dateOfB}`);
const input = await page.$("#DOB");
await input.click({ clickCount: 3 });
await input.type(dateOfB);
const response = await pollForRequestResults(config.apiKey, requestId);
console.log(`Entering recaptcha response ${response}`);
await page.evaluate(
`document.getElementById("g-recaptcha-response").innerHTML="${response}";`
);
console.log(`Submitting....`);
page.click("#Submit");
} catch (error) {
console.log(
"Your request could not be completed at this time, please check your pin number and date of birth. Also make sure your internet connection is working and try again."
);
console.error(error);
}
await page.waitForSelector(
"body > div.container.body-content > div:nth-child(1) > div:nth-child(2) > p"
);
const image = await page.$(
"body > div.container.body-content > div:nth-child(1) > div:nth-child(2) > p"
);
await image.screenshot({
path: "testResults.png",
});
await getImageText();
await page.close(); // Close the website
await browser.close(); //close browser
await deleteImage();
}
main();
//This section grabs the text off the image that was gathered from the web scraper.
async function getImageText() {
// Creates a client
const client = new vision.ImageAnnotatorClient();
console.log(`Looking for text in image`);
// Performs label detection on the image file
const [result] = await client.textDetection("./testResults.png");
const [annotation] = result.textAnnotations;
const text = annotation ? annotation.description : "";
console.log("Extracted text from image:", text);
//Pushed the text into a globally available array.
textArray.push(text);
//Sent a NOTIFICATION ALERT to the client with the text gathered from the image.
var axios = require("axios");
var data = JSON.stringify({
to: "dp8vGNkcYKb-k-72j7t4Mo:APA91bEfrI3_ht89t5X1f3_Y_DACZc9DbWI4VzcYehaQoXtD_IHIFSwm9H1hgXHNq46BQwDTlCKzkWNAHbBGauEXZNQtvhQc8glz4sHQr3JY3KM7OkUEcNB7qMMpCPxRe5GzzHbe3rkE",
notification: {
body: text,
title: "AverHealth Schedule",
},
});
var config = {
method: "post",
url: "https://fcm.googleapis.com/fcm/send",
headers: {
"Content-Type": "application/json",
Authorization: `key=${process.env.FCM_SERVER_KEY}`,
},
data: data,
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
}
//Captcha Solver for the web scraper
async function initiateCaptchaRequest(apiKey) {
const formData = {
key: apiKey,
method: "userrecaptcha",
googlekey: config.sitekey,
json: 1,
pageurl: config.pageurl,
};
console.log(
`Submitting recaptcha request to 2captcha for ${config.pageurl}`
);
const response = await request.post(config.apiSubmitUrl, {
form: formData,
});
console.log(response);
return JSON.parse(response).request;
}
async function pollForRequestResults(
key,
id,
retries = 90,
interval = 5000,
delay = 1500
) {
console.log(`Waiting for ${delay} milliseconds....`);
await timeout(delay);
return poll({
taskFn: requestCaptchaResults(key, id),
interval,
retries,
});
}
function requestCaptchaResults(apiKey, requestId) {
const url = `${config.apiRetrieveUrl}?key=${apiKey}&action=get&id=${requestId}&json=1`;
console.log(url);
return async function () {
return new Promise(async function (resolve, reject) {
console.log(`Polling for response...`);
const rawResponse = await request.get(url);
console.log(rawResponse);
const resp = JSON.parse(rawResponse);
console.log(resp);
if (resp.status === 0) return reject(resp.request);
console.log("Response received");
console.log(resp);
resolve(resp.request);
});
};
}
// DELETES THE FILE CREATED BY GOOGLEAPI
function deleteImage() {
const path = "./testResults.png";
try {
fs.unlinkSync(path);
console.log("File removed:", path);
} catch (err) {
console.error(err);
}
}
const timeout = (ms) => new Promise((res) => setTimeout(res, ms));
};
app.use(express.urlencoded({ extended: false }));
// Route to results Page
app.get("/results", (req, res) => {
res.sendFile(__dirname + "/src/results.html");
res.send(textArray);
});
app.post("/results", (req, res) => {
// Insert Login Code Here
let username = req.body.username;
let password = req.body.password;
App(username, password);
});
app.listen(port, () => {
console.log(`Scraper app listening at http://localhost:${port}`);
});