Commit 36c56f78 authored by Thomas Urban's avatar Thomas Urban

initial commit

parents
.idea
.vscode
node_modules
npm-debug.log
*.cache
{
"name": "some-tool",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"cross-env": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/cross-env/-/cross-env-5.2.0.tgz",
"integrity": "sha512-jtdNFfFW1hB7sMhr/H6rW1Z45LFqyI431m3qU6bFXcQ3Eh7LtBuG3h74o7ohHZ3crrRkkqHlo4jYHFPcjroANg==",
"requires": {
"cross-spawn": "^6.0.5",
"is-windows": "^1.0.0"
}
},
"cross-spawn": {
"version": "6.0.5",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
"integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
"requires": {
"nice-try": "^1.0.4",
"path-key": "^2.0.1",
"semver": "^5.5.0",
"shebang-command": "^1.2.0",
"which": "^1.2.9"
}
},
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"is-windows": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
"integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA=="
},
"isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"nice-try": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="
},
"path-key": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
"integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
},
"semver": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
},
"shebang-command": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
"integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
"requires": {
"shebang-regex": "^1.0.0"
}
},
"shebang-regex": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
"integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
},
"which": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
"requires": {
"isexe": "^2.0.0"
}
}
}
}
{
"name": "some-tool",
"version": "1.0.0",
"description": "",
"main": "stress-http.js",
"scripts": {
"start": "cross-env DEBUG=* node stress-http.js https://www.game.de/",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"cross-env": "^5.2.0",
"debug": "^4.1.1"
}
}
/**
* (c) 2019 cepharum GmbH, Berlin, http://cepharum.de
*
* The MIT License (MIT)
*
* Copyright (c) 2019 cepharum GmbH
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* @author: cepharum
*/
"use strict";
const MaxUrlCount = 100;
const NumClients = 20;
const DelayRange = 10;
const DelayMinimum = 10;
const File = require( "fs" );
const Path = require( "path" );
const HTTP = require( "http" );
const HTTPS = require( "https" );
const Url = require( "url" );
const { EventEmitter } = require( "events" );
const debug = require( "debug" );
const StartUrl = process.argv[2];
if ( !StartUrl ) {
console.error( `usage: ${process.argv[1]} <url>` );
process.exit( 1 );
}
const log = debug( "stress-test" );
log( "starting" );
/**
* Represents single HTTP client tracking its own set of cookies.
*/
class Client extends EventEmitter {
/**
* @param {string} clientID ID of client basically used for logging
* @param {boolean} noisy set true to get more logging, false for logging errors only
*/
constructor( clientID, noisy = true ) {
super();
this.cookies = {};
this.id = clientID;
if ( noisy ) {
this.logInfo = debug( "client " + clientID + " info ");
} else {
this.logInfo = () => {};
}
this.logError = debug( "client " + clientID + " ERROR" );
this.success = 0;
this.failed = 0;
}
/**
* Fetches provided URL using GET method.
*
* @param {string} url URL to be fetched using GET method
* @returns {Promise<{headers:object<string,string>, body:Buffer}>} promises response headers and body
*/
fetch( url ) {
this.logInfo( `fetching URL ${url}` );
return new Promise( ( resolve, reject ) => {
const options = {
headers: {
cookie: Object.keys( this.cookies ).map( name => name + "=" + this.cookies[name] ).join( ";" ),
"user-agent": `Mozilla/5.0 (stress-test.js ${this.id.trim()}) ${process.platform}`,
},
timeout: 5000,
};
const request = ( url.startsWith( "https://" ) ? HTTPS : HTTP ).get( url, options, res => {
const chunks = [];
( res.headers["set-cookie"] || [] )
.forEach( cookie => {
const match = /^\s*([^=]+)\s*=\s*([^;]*)/;
if ( match ) {
this.cookies[match[1]] = match[2];
}
} );
if ( res.statusCode >= 400 ) {
this.logError( `ERROR: fetching URL ${url} failed with ${res.statusCode}` );
reject( new Error( `fetching URL ${url} failed with ${res.statusCode}` ) );
res.destroy();
return;
}
if ( res.statusCode >= 300 ) {
res.destroy();
const location = res.headers.location;
if ( location ) {
this.logInfo( `fetching URL ${url} redirects to ${location}` );
this.fetch( location ).then( resolve, reject );
} else {
this.logError( `ERROR: fetching URL ${url} resulted in redirect w/o target` );
reject( new Error( `fetching URL ${url} resulted in redirect w/o target` ) );
}
return;
}
res.setTimeout( 20000 );
res.once( "error", error => {
this.logError( `ERROR: fetching URL ${url} failed: ${error}` );
reject( error );
} );
res.once( "timeout", () => reject( new Error( `response from URL ${url} timed out` ) ) );
res.on( "data", chunk => chunks.push( chunk ) );
res.on( "end", () => {
this.logInfo( `fetched URL ${url}` );
resolve( {
headers: res.headers,
body: Buffer.concat( chunks ),
} );
} );
} );
request.setTimeout( 20000 );
request.once( "error", reject );
request.once( "timeout", () => reject( new Error( `request for URL ${url} timed out` ) ) );
request.end();
} )
.then( data => {
this.success++;
return data;
}, error => {
this.failed++;
throw error;
} );
}
}
const cacheFile = Path.resolve( __dirname, new URL( StartUrl ).hostname + ".cache" );
new Promise( resolve => {
if ( File.existsSync( cacheFile ) ) {
log( "using URLs from cache" );
resolve( File.readFileSync( cacheFile, "utf8" ).split( /\r?\n/ ).map( line => line.trim() ) );
}
resolve( undefined );
} )
.catch( error => {
log( `reading cache file failed: ${error.message}` );
} )
.then( list => {
if ( list && Array.isArray( list ) && list.length >= 10 ) {
return list;
}
return new Promise( ( resolve, reject ) => {
const inQueue = [StartUrl];
const results = {};
const grabber = new Client( "grabber" );
grab( inQueue, results );
function grab( queue, found ) {
let url;
do {
url = queue.shift();
} while ( queue.length && !url );
if ( !url ) {
resolve();
return;
}
grabber.fetch( url )
.then( ( { body } ) => {
found[url] = true;
const numFound = Object.keys( found ).length;
grabber.logInfo( `got ${numFound}/${MaxUrlCount} URLs (queued: ${queue.length})` );
if ( numFound >= MaxUrlCount ) {
resolve( Object.keys( found ) );
return;
}
const html = body.toString( "utf8" );
const ptn = /<a\s+.*?href=(['"])(.+?)\1/ig;
let match;
while ( ( match = ptn.exec( html ) ) ) {
let newUrl = match[2];
if ( !newUrl.startsWith( StartUrl ) ) {
newUrl = Url.resolve( url, newUrl );
}
if ( !found[newUrl] && queue.indexOf( newUrl ) < 0 && newUrl.startsWith( StartUrl ) ) {
queue.push( newUrl );
}
}
if ( queue.length < 1 ) {
resolve( Object.keys( found ) );
} else {
setTimeout( grab, 100, queue, found );
}
}, error => {
if ( queue.length < 1 ) {
reject( error );
} else {
setTimeout( grab, 100, queue, found );
}
} );
}
} )
.then( list => {
File.writeFileSync( cacheFile, list.join( process.platform === "win32" ? "\r\n" : "\n" ) )
return list;
} );
} )
.then( list => {
let cancelled = false;
let finished = false;
let started = new Date();
log( `creating ${NumClients} client(s) for requesting files` );
return new Promise( resolve => {
process.on( "SIGINT", cancel );
process.on( "SIGTERM", cancel );
const clients = new Array( NumClients )
.fill( 0 )
.map( ( _, i ) => new Client( ( "#" + ( i + 1 ) ).padStart( 4 ) ) );
clients.forEach( client => {
let tid = setTimeout( fetch, randomDelay() / 100 );
client.once( "cancelled", () => {
if ( tid ) {
clearTimeout( tid );
cancelClient( client );
}
} );
function fetch() {
tid = null;
if ( cancelled ) {
cancelClient( client );
return;
}
const url = list[Math.floor( Math.random() * list.length )];
client.fetch( url )
.catch( () => {} )
.then( () => {
if ( cancelled ) {
cancelClient( client );
} else {
tid = setTimeout( fetch, randomDelay() );
}
} );
}
} );
function randomDelay() {
return Math.round( Math.random() * DelayRange * 1000 ) + DelayMinimum * 1000;
}
function cancel() {
log( "terminating..." );
cancelled = cancelled || {};
clients.forEach( client => client.emit( "cancelled" ) );
}
function cancelClient( client ) {
cancelled[client.id] = true;
if ( Object.keys( cancelled ).length >= NumClients ) {
resolve( clients );
}
}
} )
.then( clients => {
const ended = new Date();
const elapsed = Math.round( ( ended.getTime() - started.getTime() ) / 1000 );
log( `Results after ${elapsed} seconds (${Math.round( elapsed / 60 )} minutes) running ${NumClients} clients` );
const total = { success: 0, failed: 0 };
clients.forEach( client => {
log( `client ${client.id} success: ${String( client.success ).padStart( 6 )} - failed: ${String( client.failed ).padStart( 6 )}` );
total.success += client.success;
total.failed += client.failed;
} );
log( `TOTAL success: ${String( total.success ).padStart( 6 )} - failed: ${String( total.failed ).padStart( 6 )}` );
finished = true;
} );
} )
.catch( error => {
console.error( error );
} );
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment