diff --git a/client/src/App.js b/client/src/App.js
index 96914d7..dc6c929 100644
--- a/client/src/App.js
+++ b/client/src/App.js
@@ -5,33 +5,34 @@ import './App.css';
function App() {
const [response, setResponse] = useState({});
- const [show, setShow] = useState(false);
+ const [loading, setLoading] = useState(false);
const [fetchError, setFetchError] = useState('');
- const [pdfBlob, setPdfBlob] = useState(null);
+ const [buffer, setBuffer] = useState(null);
- const loadURL = (url, width, height) => {
- setShow(true);
- fetch(
- `http://127.0.0.1:3001/website?url=${url}&width=${width}&height=${height}`
- )
- .then(async (response) => {
- if (response.ok) {
- const json = await response.json();
- if (json.data.url) {
- json.data.url = `http://localhost:3000/redirect/${json.data.url}`;
- }
- setResponse({
- url: json.data.url,
- width,
- height,
- thumb: json.data.thumb,
- origUrl: url
- });
- setShow(false);
+ const PORT = 3100;
+ const PATH = `0.0.0.0:${PORT}`;
+
+
+ const loadURL = (url) => {
+ setLoading(true);
+ setFetchError('');
+ fetch(`http://${PATH}/website-proxy-pdftron?url=${url}`)
+ .then(async (res) => {
+ var size = { width: 1800, height: 7000 };
+ try {
+ size = JSON.parse(res.statusText);
+ } catch (e) {
}
+ setResponse({
+ url: `http://${PATH}`,
+ thumb: '',
+ ...size,
+ origUrl: `http://${PATH}`,
+ });
+ setLoading(false);
})
.catch((err) => {
- setShow(false);
+ setLoading(false);
setFetchError(
'Trouble fetching the URL, please make sure the server is running. `cd server && npm start`'
);
@@ -39,30 +40,29 @@ function App() {
};
const downloadPDF = () => {
- setShow(true);
if (response.url) {
- const urlArray = response.url.split('/');
- fetch(
- `http://127.0.0.1:3001/getpdf?url=${urlArray[4]}&width=${response.width}&height=${response.height}`
- ).then((res) => {
- return res.blob('application/pdf');
- }).then((blob) => {
- console.log(blob);
- setPdfBlob(blob);
- });
+ setLoading(true);
+ fetch(`http://${PATH}/pdftron-pdf`)
+ .then(async (res) => {
+ console.log(res);
+ setBuffer(res);
+ setLoading(false);
+ }).catch(err => console.log(err));
}
- setShow(false);
};
+ // receive new loading state from Viewer as loadDocAndAnnots takes a while
+ const loadingFromViewer = (value) => setLoading(value);
+
return (
);
}
diff --git a/client/src/components/navigation/Nav.js b/client/src/components/navigation/Nav.js
index 14b51ba..15767a9 100644
--- a/client/src/components/navigation/Nav.js
+++ b/client/src/components/navigation/Nav.js
@@ -14,54 +14,39 @@ import './Nav.css';
const Nav = ({ handleSubmit, fetchError, showSpinner, handleDownload }) => {
const [url, setUrl] = useState('');
- const [width, setWidth] = useState(1000);
- const [height, setHeight] = useState(2000);
const [error, setError] = useState(false);
+ // test is URL (without https://) is valid https://regexr.com/3e6m0
+ const isValidURL = (url) => {
+ // eslint-disable-next-line no-useless-escape
+ return /(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)/gi.test(url);
+ }
+
return (
WebViewer HTML
- In this demo, you can pass any URL. The URL passed in will be scraped
- and saved server-side as a snapshot in time. Then you will be annotate
- that copy here.
+ In this demo, you can pass any URL. The URL passed in will be proxied
+ and you will be able to annotate directly here.
URL of the page
{
- setUrl(`http://${e.target.value}`);
+ setUrl(e.target.value);
}}
>
-
-
- Width of the page
-
-
- Height of the page
-
{
- if (url !== '' && width !== 0 && height !== 0) {
- handleSubmit(url, width, height);
+ setError(false);
+ if (!!url && isValidURL(url)) {
+ handleSubmit(`https://${url}`);
} else {
setError(true);
}
@@ -70,8 +55,6 @@ const Nav = ({ handleSubmit, fetchError, showSpinner, handleDownload }) => {
{showSpinner &&
-
-
handleDownload()}>
{showSpinner &&
- Please enter a valid URL, width and height and try again.
+ Please enter a valid URL try again.
)}
{fetchError ? {fetchError} : null}
diff --git a/client/src/components/viewer/Viewer.js b/client/src/components/viewer/Viewer.js
index ca213cd..ca8b27a 100644
--- a/client/src/components/viewer/Viewer.js
+++ b/client/src/components/viewer/Viewer.js
@@ -3,10 +3,11 @@ import { initializeHTMLViewer } from '@pdftron/webviewer-html';
import { useEffect, useRef, useState } from 'react';
import './Viewer.css';
-const Viewer = ({ res, loadURL, pdf }) => {
+const Viewer = ({ res, loadURL, buffer, loading }) => {
const viewer = useRef(null);
const [HTMLModule, setHTMLModule] = useState(null);
const [instance, setInstance] = useState(null);
+ const [size, setSize] = useState({});
useEffect(() => {
WebViewer(
@@ -17,7 +18,7 @@ const Viewer = ({ res, loadURL, pdf }) => {
).then(async (instance) => {
const { FitMode, docViewer } = instance;
setInstance(instance);
- instance.setFitMode(FitMode.FitPage);
+ instance.setFitMode(FitMode.FitWidth);
// disable some incompatible tools
instance.disableElements([
'highlightToolGroupButton',
@@ -35,13 +36,15 @@ const Viewer = ({ res, loadURL, pdf }) => {
docViewer.on('documentLoaded', () => {
setTimeout(() => {
- instance.setFitMode(FitMode.FitPage);
+ if (instance.getFitMode() !== FitMode.FitWidth) {
+ instance.setFitMode(FitMode.FitWidth);
+ }
}, 1500);
});
setHTMLModule(htmlModule);
- loadURL(`https://www.pdftron.com/`, 1800, 1100);
+ loadURL(`https://www.pdftron.com/`);
});
// eslint-disable-next-line
}, []);
@@ -49,33 +52,41 @@ const Viewer = ({ res, loadURL, pdf }) => {
useEffect(() => {
if (HTMLModule && Object.keys(res).length > 0) {
const { url, width, height, thumb, origUrl } = res;
+ setSize({width, height});
HTMLModule.loadHTMLPage({ url, width, height, thumb, origUrl });
}
}, [HTMLModule, res]);
useEffect(() => {
const loadDocAndAnnots = async () => {
- const doc = await instance.CoreControls.createDocument(pdf);
+ loading(true);
+ const doc = await instance.Core.createDocument(buffer, {
+ extension: 'png',
+ pageSizes: [size],
+ });
+
+ // exportAnnotations as xfdfString seem to misplace annotations
const xfdf = await instance.docViewer
.getAnnotationManager()
.exportAnnotations();
const data = await doc.getFileData({ xfdfString: xfdf });
- const arr = new Uint8Array(data);
- const blob = new Blob([arr], { type: 'application/pdf' });
+ const blob = new Blob([data], { type: 'application/pdf' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'annotated';
a.click();
a.remove();
+ loading(false);
// in case the Blob uses a lot of memory
- setTimeout(() => URL.revokeObjectURL(url), 7000);
+ setTimeout(() => URL.revokeObjectURL(url), 5000);
};
- if (instance && pdf) {
+ if (instance && buffer) {
loadDocAndAnnots();
}
- }, [instance, pdf]);
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [instance, buffer]);
return
;
};
diff --git a/client/src/setupProxy.js b/client/src/setupProxy.js
deleted file mode 100644
index a024ce8..0000000
--- a/client/src/setupProxy.js
+++ /dev/null
@@ -1,12 +0,0 @@
-const { createProxyMiddleware } = require('http-proxy-middleware');
-
-module.exports = function (app) {
- app.use(
- '/redirect',
- createProxyMiddleware({
- target: 'http://localhost:3001',
- changeOrigin: true,
- pathRewrite: { '^/redirect': '' },
- })
- );
-};
diff --git a/server/index.js b/server/index.js
index e88ebda..367843a 100644
--- a/server/index.js
+++ b/server/index.js
@@ -1,93 +1,202 @@
+// TAKEN FROM: https://stackoverflow.com/a/63602976
const express = require('express');
const cors = require('cors');
-const scrape = require('website-scraper');
-const puppeteer = require('puppeteer');
-const path = require('path');
-const fs = require('fs');
-
-const PORT = 3001;
+const https = require('https');
+const http = require('http');
const app = express();
-
-app.use(express.static('public'));
app.use(cors());
-// endpoint to scrape the website and generate a thumb preview
-app.get('/website', (req, res) => {
- const { url, width, height } = req.query;
- const timestamp = Date.now();
- if (!url) {
- res.status(400).json({
- status: 'Bad Request',
- data: 'Please provide URL of the website you want to scrape as a query parameter.',
- });
+const puppeteer = require('puppeteer');
+
+const PORT = 3100;
+const PATH = `0.0.0.0:${PORT}`;
+
+const getHostPortSSL = (url) => {
+ const parsedHost = url.split('/').splice(2).splice(0, 1).join('/')
+ let parsedPort;
+ let parsedSSL;
+ if (url.startsWith('https://')) {
+ parsedPort = 443;
+ parsedSSL = https;
+ } else if (url.startsWith('http://')) {
+ parsedPort = 80;
+ parsedSSL = http;
}
- const urlToConvert = new URL(url);
- const pagePath = `${urlToConvert.hostname}${timestamp}`;
- const directory = path.resolve(__dirname, `./public/${pagePath}`);
-
- const options = {
- urls: [url],
- directory,
- filenameGenerator: `${pagePath}`,
- };
-
- scrape(options).then(async (result) => {
- // get the screenshot with puppeteer after the scrape is complete
- const browser = await puppeteer.launch({
- defaultViewport: {
- width: Number(width),
- height: Number(height),
- },
- });
- const page = await browser.newPage();
- await page.goto(`http://127.0.0.1:${PORT}/${pagePath}/index.html`);
- const thumbPath = path.resolve(__dirname, `./public/${pagePath}/thumb.png`);
- await page.screenshot({
- path: thumbPath,
- });
+ return {
+ parsedHost,
+ parsedPort,
+ parsedSSL,
+ }
+}
- // read the file from the filepath and respond to server with URL and thumb
- await fs.readFile(thumbPath, { encoding: 'base64' }, (err, data) => {
- if (err) throw err;
- const prefix = 'data:image/png;base64,';
- res.status(200).json({
- status: 'success',
- data: {
- url: `${pagePath}/index.html`,
- thumb: prefix + data,
- },
- });
- });
+const isUrlAbsolute = (url) => (url.indexOf('://') > 0 || url.indexOf('//') === 0);
+const isUrlNested = (url) => {
+ let nested = url.split('/').splice(3);
+ if (nested.length > 0 && nested[0] != '') {
+ return true;
+ }
+ return false;
+}
+
+const defaultViewport = { width: 1680, height: 1050 };
+var url;
+var dimensions;
+
+app.get('/website-proxy-pdftron', async function (req, res, next) {
+ // this is the url retrieved from the input
+ url = req.query.url;
+ console.log('\x1b[31m%s\x1b[0m', `
+ ***********************************************************************
+ ************************** NEW REQUEST ********************************
+ ***********************************************************************
+ `);
- await browser.close();
+ const browser = await puppeteer.launch({
+ defaultViewport,
+ headless: true,
});
+ const page = await browser.newPage();
+ await page.goto(url);
+
+ // Get the "viewport" of the page, as reported by the page.
+ dimensions = await page.evaluate(() => {
+ return {
+ width: document.body.clientWidth,
+ height: document.body.clientHeight,
+ };
+ });
+
+ await browser.close();
+
+ // next("router") pass control to next route and strip all req.query, if queried url contains nested route this will be lost in subsequest requests
+ next();
});
-app.get('/getpdf', async (req, res) => {
- const { url, width, height } = req.query;
- const pagePath = path.resolve(__dirname, `./public/pdf/html.pdf`);
+// need to be placed before app.use('/');
+app.get('/pdftron-pdf', async (req, res) => {
const browser = await puppeteer.launch({
- defaultViewport: {
- width: Number(width),
- height: Number(height),
- },
+ defaultViewport,
+ headless: true,
});
const page = await browser.newPage();
- await page.goto(`http://127.0.0.1:${PORT}/${url}/index.html`);
- await page.pdf({
- path: pagePath,
- width: Number(width),
- height: Number(height),
- printBackground: true,
- pageRanges: '1',
+ await page.goto(`http://${PATH}`, {
+ waitUntil: 'networkidle0'
});
+
+ const buffer = await page.screenshot({ type: 'png', fullPage: true });
await browser.close();
// read the file from the filepath and respond to server
- res.sendFile(pagePath);
+ res.send(buffer);
});
-app.listen(PORT, () => {
- console.log(`Server is now live at ${PORT}`);
-});
+app.use('/', function(clientRequest, clientResponse) {
+ const {
+ parsedHost,
+ parsedPort,
+ parsedSSL,
+ } = getHostPortSSL(url);
+
+ // convert to original url, since clientRequest.url starts from /website-proxy-pdftron and will be redirected
+ if (clientRequest.url.startsWith('/website-proxy-pdftron')) {
+ clientRequest.url = url;
+ }
+
+ // if url has nested route then convert to original url to force request it
+ // did not work with nested urls from developer.mozilla.org
+ // check if nested route cause instagram.com doesn't like this
+ if (isUrlNested(url) && clientRequest.url === '/') {
+ clientRequest.url = url;
+ }
+
+ var options = {
+ hostname: parsedHost,
+ port: parsedPort,
+ path: clientRequest.url,
+ method: clientRequest.method,
+ headers: {
+ 'User-Agent': clientRequest.headers['user-agent']
+ }
+ };
+ console.log('hostname', options.hostname, 'path', options.path);
+
+ const callback = (serverResponse, clientResponse) => {
+ // Delete 'x-frame-options': 'SAMEORIGIN'
+ // so that the page can be loaded in an iframe
+ delete serverResponse.headers['x-frame-options'];
+ delete serverResponse.headers['content-security-policy'];
+
+ // if a url is blown up, make sure to reset cache-control
+ if (!!serverResponse.headers['cache-control'] && /max-age=[^0]/.test(String(serverResponse.headers['cache-control']))) {
+ serverResponse.headers['cache-control'] = 'max-age=0';
+ }
+ var body = '';
+ if (String(serverResponse.headers['content-type']).indexOf('text/html') !== -1) {
+ serverResponse.on('data', function (chunk) {
+ body += chunk;
+ });
+
+ serverResponse.on('end', function () {
+ // Make changes to HTML files when they're done being read.
+ body = body.replace(`example`, `Cat!`);
+
+ // can also send dimensions in clientResponse.setHeader() but for some reason, on client can't read response.headers.get() but it's available in the network tab
+ clientResponse.writeHead(serverResponse.statusCode, JSON.stringify(dimensions), serverResponse.headers);
+ clientResponse.end(body);
+ });
+ }
+ else {
+ serverResponse.pipe(clientResponse, {
+ end: true
+ });
+ // Can be undefined
+ if (serverResponse.headers['content-type']) {
+ clientResponse.contentType(serverResponse.headers['content-type'])
+ }
+ }
+ }
+
+ var serverRequest = parsedSSL.request(options, serverResponse => {
+ console.log('serverResponse', serverResponse.statusCode, serverResponse.headers)
+ // This is the case of urls being redirected -> retrieve new headers['location'] and request again
+ if (serverResponse.statusCode > 299 && serverResponse.statusCode < 400) {
+ var location = serverResponse.headers['location'];
+ var parsedLocation = isUrlAbsolute(location) ? location : `https://${parsedHost}${location}`;
+
+ const {
+ parsedHost: newParsedHost,
+ parsedPort: newParsedPort,
+ parsedSSL: newParsedSSL,
+ } = getHostPortSSL(parsedLocation);
+
+ var newOptions = {
+ hostname: newParsedHost,
+ port: newParsedPort,
+ path: parsedLocation,
+ method: clientRequest.method,
+ headers: {
+ 'User-Agent': clientRequest.headers['user-agent']
+ }
+ };
+ console.log('newhostname', newOptions.hostname, 'newpath', newOptions.path);
+
+ var newServerRequest = newParsedSSL.request(newOptions, newResponse => {
+ console.log('new serverResponse', newResponse.statusCode, newResponse.headers)
+ callback(newResponse, clientResponse);
+ });
+ serverRequest.end();
+ newServerRequest.end();
+ return;
+ }
+
+ callback(serverResponse, clientResponse);
+ });
+
+ serverRequest.end();
+
+ });
+
+
+ app.listen(PORT);
+ console.log(`Running on ${PATH}`);
\ No newline at end of file
diff --git a/server/package-lock.json b/server/package-lock.json
index 36f1632..2a47149 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -95,6 +95,14 @@
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz",
"integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA=="
},
+ "axios": {
+ "version": "0.23.0",
+ "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz",
+ "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==",
+ "requires": {
+ "follow-redirects": "^1.14.4"
+ }
+ },
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
@@ -268,6 +276,41 @@
"vary": "^1"
}
},
+ "cors-anywhere": {
+ "version": "0.4.4",
+ "resolved": "https://registry.npmjs.org/cors-anywhere/-/cors-anywhere-0.4.4.tgz",
+ "integrity": "sha512-8OBFwnzMgR4mNrAeAyOLB2EruS2z7u02of2bOu7i9kKYlZG+niS7CTHLPgEXKWW2NAOJWRry9RRCaL9lJRjNqg==",
+ "requires": {
+ "http-proxy": "1.11.1",
+ "proxy-from-env": "0.0.1"
+ },
+ "dependencies": {
+ "eventemitter3": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-1.2.0.tgz",
+ "integrity": "sha1-HIaZHYFq0eUEdQ5zh0Ik7PO+xQg="
+ },
+ "http-proxy": {
+ "version": "1.11.1",
+ "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.11.1.tgz",
+ "integrity": "sha1-cd9VdX6ALVjqgQ3yJEAZ3aBa6F0=",
+ "requires": {
+ "eventemitter3": "1.x.x",
+ "requires-port": "0.x.x"
+ }
+ },
+ "proxy-from-env": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-0.0.1.tgz",
+ "integrity": "sha1-snxJRunm1dutt1mKZDXTAUxM/Uk="
+ },
+ "requires-port": {
+ "version": "0.0.1",
+ "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-0.0.1.tgz",
+ "integrity": "sha1-S0QUQR2d98hVmV3YmajHiilRwW0="
+ }
+ }
+ },
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
@@ -525,6 +568,11 @@
"path-exists": "^4.0.0"
}
},
+ "follow-redirects": {
+ "version": "1.14.4",
+ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz",
+ "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g=="
+ },
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@@ -541,9 +589,9 @@
}
},
"forwarded": {
- "version": "0.1.2",
- "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
- "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
+ "version": "0.2.0",
+ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+ "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="
},
"fresh": {
"version": "0.5.2",
@@ -640,6 +688,16 @@
"toidentifier": "1.0.0"
}
},
+ "http-proxy": {
+ "version": "1.18.1",
+ "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz",
+ "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==",
+ "requires": {
+ "eventemitter3": "^4.0.0",
+ "follow-redirects": "^1.0.0",
+ "requires-port": "^1.0.0"
+ }
+ },
"http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -1015,11 +1073,11 @@
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
},
"proxy-addr": {
- "version": "2.0.6",
- "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
- "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
+ "version": "2.0.7",
+ "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+ "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
"requires": {
- "forwarded": "~0.1.2",
+ "forwarded": "0.2.0",
"ipaddr.js": "1.9.1"
}
},
@@ -1146,6 +1204,11 @@
}
}
},
+ "requires-port": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
+ "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
+ },
"rimraf": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
diff --git a/server/package.json b/server/package.json
index 2155cb6..ed5c8d5 100644
--- a/server/package.json
+++ b/server/package.json
@@ -9,8 +9,11 @@
"author": "Andrey Safonov",
"license": "ISC",
"dependencies": {
+ "axios": "^0.23.0",
"cors": "^2.8.5",
+ "cors-anywhere": "^0.4.3",
"express": "^4.17.1",
+ "http-proxy": "^1.18.1",
"puppeteer": "^7.0.4",
"website-scraper": "^4.2.3"
},