From f862c362fe1454c5f163c396d509c7c40b56a927 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 23 Nov 2021 15:09:53 +0100 Subject: [PATCH 001/109] npm init --- package-lock.json | 28 ++++++++++++++++++++++++++++ package.json | 18 ++++++++++++++++++ src/main.js | 0 3 files changed, 46 insertions(+) create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 src/main.js diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..2230719 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,28 @@ +{ + "name": "rfmon-to-influx", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "rfmon-to-influx", + "version": "1.0.0", + "license": "AGPL-3.0", + "dependencies": { + "@influxdata/influxdb-client": "^1.20.0" + } + }, + "node_modules/@influxdata/influxdb-client": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.20.0.tgz", + "integrity": "sha512-jaKSI63hmQ5VSkJrFJkYIXaKlhoF+mGd4HmOf7v/X7pmEi69ReHp922Wyx6/OeCrpndRMbsadk+XmGNdd43cFw==" + } + }, + "dependencies": { + "@influxdata/influxdb-client": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.20.0.tgz", + "integrity": "sha512-jaKSI63hmQ5VSkJrFJkYIXaKlhoF+mGd4HmOf7v/X7pmEi69ReHp922Wyx6/OeCrpndRMbsadk+XmGNdd43cFw==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..04bfc2e --- /dev/null +++ b/package.json @@ -0,0 +1,18 @@ +{ + "name": "rfmon-to-influx", + "version": "1.0.0", + "description": "Writing (mostly meta-) data received in Wireless-Monitor-Mode into an InfluxDB", + "main": "src/main.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "https://gitea.ruekov.eu/Ruakij/rfmon-to-influx" + }, + "author": "Ruakij", + "license": "AGPL-3.0", + "dependencies": { + "@influxdata/influxdb-client": "^1.20.0" + } +} diff --git a/src/main.js b/src/main.js new file mode 100644 index 0000000..e69de29 From 6ccd02af6c46b12bbff59d7d2a8b2d3365df8272 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 23 Nov 2021 15:14:55 +0100 Subject: [PATCH 002/109] Added start-script --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 04bfc2e..90d0c54 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "description": "Writing (mostly meta-) data received in Wireless-Monitor-Mode into an InfluxDB", "main": "src/main.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node src/main.js" }, "repository": { "type": "git", From 20ed3d0928f8522ee551d1bc3aaca9fb1c3b49c9 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 23 Nov 2021 15:15:10 +0100 Subject: [PATCH 003/109] Docker setup --- .dockerignore | 3 +++ Dockerfile | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..30a9de6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +node_modules +npm-debug.log +.* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..98ec383 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM node:16 + +# Create app directory +WORKDIR /usr/src/app + +# Install app dependencies +COPY package*.json ./ +RUN npm install + +# Bundle app source +COPY ./src/ . + +CMD ["npm", "run"] \ No newline at end of file From 7562ff2268177325891c12b1ff3301ed1c581a7f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 23 Nov 2021 17:05:43 +0100 Subject: [PATCH 004/109] Added module log4js --- package-lock.json | 197 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 3 +- src/main.js | 6 ++ 3 files changed, 204 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2230719..68a64c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,13 +9,123 @@ "version": "1.0.0", "license": "AGPL-3.0", "dependencies": { - "@influxdata/influxdb-client": "^1.20.0" + "@influxdata/influxdb-client": "^1.20.0", + "log4js": "^6.3.0" } }, "node_modules/@influxdata/influxdb-client": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.20.0.tgz", "integrity": "sha512-jaKSI63hmQ5VSkJrFJkYIXaKlhoF+mGd4HmOf7v/X7pmEi69ReHp922Wyx6/OeCrpndRMbsadk+XmGNdd43cFw==" + }, + "node_modules/date-format": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-3.0.0.tgz", + "integrity": "sha512-eyTcpKOcamdhWJXj56DpQMo1ylSQpcGtGKXcU0Tb97+K56/CF5amAqqqNj0+KvA0iw2ynxtHWFsPDSClCxe48w==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==" + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/log4js": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.3.0.tgz", + "integrity": "sha512-Mc8jNuSFImQUIateBFwdOQcmC6Q5maU0VVvdC2R6XMb66/VnT+7WS4D/0EeNMZu1YODmJe5NIn2XftCzEocUgw==", + "dependencies": { + "date-format": "^3.0.0", + "debug": "^4.1.1", + "flatted": "^2.0.1", + "rfdc": "^1.1.4", + "streamroller": "^2.2.4" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, + "node_modules/streamroller": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-2.2.4.tgz", + "integrity": "sha512-OG79qm3AujAM9ImoqgWEY1xG4HX+Lw+yY6qZj9R1K2mhF5bEmQ849wvrb+4vt4jLMLzwXttJlQbOdPOQVRv7DQ==", + "dependencies": { + "date-format": "^2.1.0", + "debug": "^4.1.1", + "fs-extra": "^8.1.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/streamroller/node_modules/date-format": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-2.1.0.tgz", + "integrity": "sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "engines": { + "node": ">= 4.0.0" + } } }, "dependencies": { @@ -23,6 +133,91 @@ "version": "1.20.0", "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.20.0.tgz", "integrity": "sha512-jaKSI63hmQ5VSkJrFJkYIXaKlhoF+mGd4HmOf7v/X7pmEi69ReHp922Wyx6/OeCrpndRMbsadk+XmGNdd43cFw==" + }, + "date-format": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-3.0.0.tgz", + "integrity": "sha512-eyTcpKOcamdhWJXj56DpQMo1ylSQpcGtGKXcU0Tb97+K56/CF5amAqqqNj0+KvA0iw2ynxtHWFsPDSClCxe48w==" + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + } + }, + "flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" + }, + "fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==" + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "log4js": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.3.0.tgz", + "integrity": "sha512-Mc8jNuSFImQUIateBFwdOQcmC6Q5maU0VVvdC2R6XMb66/VnT+7WS4D/0EeNMZu1YODmJe5NIn2XftCzEocUgw==", + "requires": { + "date-format": "^3.0.0", + "debug": "^4.1.1", + "flatted": "^2.0.1", + "rfdc": "^1.1.4", + "streamroller": "^2.2.4" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, + "streamroller": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-2.2.4.tgz", + "integrity": "sha512-OG79qm3AujAM9ImoqgWEY1xG4HX+Lw+yY6qZj9R1K2mhF5bEmQ849wvrb+4vt4jLMLzwXttJlQbOdPOQVRv7DQ==", + "requires": { + "date-format": "^2.1.0", + "debug": "^4.1.1", + "fs-extra": "^8.1.0" + }, + "dependencies": { + "date-format": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-2.1.0.tgz", + "integrity": "sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==" + } + } + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" } } } diff --git a/package.json b/package.json index 90d0c54..7bc22eb 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "author": "Ruakij", "license": "AGPL-3.0", "dependencies": { - "@influxdata/influxdb-client": "^1.20.0" + "@influxdata/influxdb-client": "^1.20.0", + "log4js": "^6.3.0" } } diff --git a/src/main.js b/src/main.js index e69de29..8602d39 100644 --- a/src/main.js +++ b/src/main.js @@ -0,0 +1,6 @@ +const log4js = require("log4js"); + +const logger = log4js.getLogger(); +logger.level = process.env.logLevel || "INFO"; + + From 184432f642468ba996c9543bfc7d4ef34ef765fc Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:03:40 +0100 Subject: [PATCH 005/109] Added helper for env-vars --- src/helper/env.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/helper/env.js diff --git a/src/helper/env.js b/src/helper/env.js new file mode 100644 index 0000000..08bff89 --- /dev/null +++ b/src/helper/env.js @@ -0,0 +1,13 @@ +function requireEnvVars(requiredEnv){ + // Ensure required ENV vars are set + let unsetEnv = requiredEnv.filter((env) => !(typeof process.env[env] !== 'undefined')); + + if (unsetEnv.length > 0) { + return "Required ENV variables are not set: [" + unsetEnv.join(', ') + "]"; + } +} + +// Specify exports +module.exports = { + requireEnvVars +}; \ No newline at end of file From 93f78aafe08088d2e8b032f23fd43fc0d96f44e1 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:06:09 +0100 Subject: [PATCH 006/109] Changed || to ?? to detect undefined env --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 8602d39..2680701 100644 --- a/src/main.js +++ b/src/main.js @@ -1,6 +1,6 @@ const log4js = require("log4js"); const logger = log4js.getLogger(); -logger.level = process.env.logLevel || "INFO"; +logger.level = process.env.logLevel ?? "INFO"; From 34c73c82c82303c8ed06fee9f4f92169e056ac96 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:06:29 +0100 Subject: [PATCH 007/109] Gave logger name --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 2680701..779804d 100644 --- a/src/main.js +++ b/src/main.js @@ -1,6 +1,6 @@ const log4js = require("log4js"); -const logger = log4js.getLogger(); +const logger = log4js.getLogger("main"); logger.level = process.env.logLevel ?? "INFO"; From d13df7b03d9c0a171a6685206559e3193e008ca2 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:06:44 +0100 Subject: [PATCH 008/109] Add comment for logger --- src/main.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.js b/src/main.js index 779804d..321de75 100644 --- a/src/main.js +++ b/src/main.js @@ -1,5 +1,6 @@ const log4js = require("log4js"); +/// Setup logger const logger = log4js.getLogger("main"); logger.level = process.env.logLevel ?? "INFO"; From 1b6721d24225d3a50ff625345fe5ed2ea2feaadb Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:07:29 +0100 Subject: [PATCH 009/109] Implemented env defaults and checking --- src/main.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main.js b/src/main.js index 321de75..152f6c8 100644 --- a/src/main.js +++ b/src/main.js @@ -1,7 +1,25 @@ const log4js = require("log4js"); +const { requireEnvVars } = require("./helper/env.js"); /// Setup logger const logger = log4js.getLogger("main"); logger.level = process.env.logLevel ?? "INFO"; +/// Setup ENVs +const env = process.env; +// Defaults +{ + env.WIFI_INTERFACE ??= "wlan0"; + env.WIFI_CHANNEL ??= [1,6,11]; + env.WIFI_CHANNEL_TIME ??= 1; +} +// Required vars +let errorMsg = requireEnvVars([ + "INFLUX_URL", "INFLUX_TOKEN", + "INFLUX_ORG", "INFLUX_BUCKET" +]); +if(errorMsg){ + logger.fatal(errorMsg); + exit(1); +} From 7dd131d6b54c408d08bfe757f71dda6342df546d Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:11:42 +0100 Subject: [PATCH 010/109] Implemented dto-obj Packet --- src/dto/Packet.js | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/dto/Packet.js diff --git a/src/dto/Packet.js b/src/dto/Packet.js new file mode 100644 index 0000000..57a0120 --- /dev/null +++ b/src/dto/Packet.js @@ -0,0 +1,46 @@ +const PacketType = { + Beacon: 'Beacon', + ProbeRequest: 'ProbeRequest', + ProbeResponse: 'ProbeResponse', + Data: 'Data', + RequestToSend: 'RequestToSend', + ClearToSend: 'ClearToSend', + Acknowledgment: 'Acknowledgment', + BlockAcknowledgment: 'BlockAcknowledgment', + NoData: 'NoData', + Unknown: 'Unknown' +} + +class Packet{ + timestampMicros; + + srcMac; + dstMac; + bssid; + + signal; + frequency; + dataRate; + + packetType; +} + +// Extensions of Packet +class PacketWithSSID extends Packet{ + ssid; +} + +class BeaconPacket extends PacketWithSSID{} +class ProbeRequestPacket extends PacketWithSSID{} +class ProbeResponsePacket extends PacketWithSSID{} + + +// Specify exports +module.exports = { + PacketType, + Packet, + PacketWithSSID, + BeaconPacket, + ProbeRequestPacket, + ProbeResponsePacket +}; From 79efcc3951fe68d1f06a574fc05717d43564d9f8 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:12:13 +0100 Subject: [PATCH 011/109] Fixed missing import --- src/main.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.js b/src/main.js index 152f6c8..9512d97 100644 --- a/src/main.js +++ b/src/main.js @@ -1,5 +1,6 @@ const log4js = require("log4js"); const { requireEnvVars } = require("./helper/env.js"); +const { exit } = require("process"); /// Setup logger const logger = log4js.getLogger("main"); From 8ee189f52ba3aaa9e0d04473d04403171eae309f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 01:16:37 +0100 Subject: [PATCH 012/109] Fixed wrong capitalisation --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 9512d97..586503c 100644 --- a/src/main.js +++ b/src/main.js @@ -4,7 +4,7 @@ const { exit } = require("process"); /// Setup logger const logger = log4js.getLogger("main"); -logger.level = process.env.logLevel ?? "INFO"; +logger.level = process.env.LOGLEVEL ?? "INFO"; /// Setup ENVs const env = process.env; From cc8b106157f4f7df6cdb7108ed7ad912d595473b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 13:10:41 +0100 Subject: [PATCH 013/109] Added logger-setup-module --- src/helper/logger.js | 11 +++++++++++ src/main.js | 8 +++----- 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 src/helper/logger.js diff --git a/src/helper/logger.js b/src/helper/logger.js new file mode 100644 index 0000000..6e61ab0 --- /dev/null +++ b/src/helper/logger.js @@ -0,0 +1,11 @@ +const log4js = require("log4js"); + + +function setup(category = "unknown"){ + const logger = log4js.getLogger(category); + logger.level = process.env.LOGLEVEL ?? "INFO"; + return logger; +} + +// Specify exports +module.exports = setup; \ No newline at end of file diff --git a/src/main.js b/src/main.js index 586503c..c1569b7 100644 --- a/src/main.js +++ b/src/main.js @@ -1,15 +1,13 @@ -const log4js = require("log4js"); +const logger = require("./helper/logger.js")("main"); + const { requireEnvVars } = require("./helper/env.js"); const { exit } = require("process"); -/// Setup logger -const logger = log4js.getLogger("main"); -logger.level = process.env.LOGLEVEL ?? "INFO"; - /// Setup ENVs const env = process.env; // Defaults { + env.LOGLEVEL ??= "INFO"; env.WIFI_INTERFACE ??= "wlan0"; env.WIFI_CHANNEL ??= [1,6,11]; env.WIFI_CHANNEL_TIME ??= 1; From 9c28ed53e38e170288354395a54addd0b5cc7799 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 13:11:01 +0100 Subject: [PATCH 014/109] Added exec-module --- src/helper/exec.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/helper/exec.js diff --git a/src/helper/exec.js b/src/helper/exec.js new file mode 100644 index 0000000..b618e3e --- /dev/null +++ b/src/helper/exec.js @@ -0,0 +1,22 @@ +const logger = require("./logger.js")("exec"); + +const { spawn } = require("child_process"); + + +function exec(cmd, stdout_handler, stderr_handler, exit_handler){ + const [bin, ...args] = cmd.split(' ') + + logger.addContext("binary", "bin"); + logger.debug(`Spawn process '${cmd}'`); + let proc = spawn(bin, args); + + logger.debug(`Attach stdout, stderr and exit-handler if set`); + stdout_handler && proc.stdout.on('data', stdout_handler); + stderr_handler && proc.stderr.on('data', stderr_handler); + exit_handler && proc.on('exit', exit_handler); +} + +// Specify exports +module.exports = { + exec +}; \ No newline at end of file From fa8d630283145759e7d62ee51e4c1eacc088e417 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 15:35:05 +0100 Subject: [PATCH 015/109] Change exec to just return process and streams --- src/helper/exec.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/helper/exec.js b/src/helper/exec.js index b618e3e..63f6c69 100644 --- a/src/helper/exec.js +++ b/src/helper/exec.js @@ -3,17 +3,18 @@ const logger = require("./logger.js")("exec"); const { spawn } = require("child_process"); -function exec(cmd, stdout_handler, stderr_handler, exit_handler){ +function exec(cmd, stdout, stderr, exit_handler){ const [bin, ...args] = cmd.split(' ') logger.addContext("binary", "bin"); logger.debug(`Spawn process '${cmd}'`); let proc = spawn(bin, args); - logger.debug(`Attach stdout, stderr and exit-handler if set`); - stdout_handler && proc.stdout.on('data', stdout_handler); - stderr_handler && proc.stderr.on('data', stderr_handler); - exit_handler && proc.on('exit', exit_handler); + return { + "process": proc, + "stdout": proc.stdout, + "stderr": proc.stderr + } } // Specify exports From d11aac559974d6ad61be9efe578f60b0c17da57b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 15:35:29 +0100 Subject: [PATCH 016/109] Implemented RegexBlockStream --- src/streamHandler/RegexBlockStream.js | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/streamHandler/RegexBlockStream.js diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js new file mode 100644 index 0000000..f06bf6d --- /dev/null +++ b/src/streamHandler/RegexBlockStream.js @@ -0,0 +1,65 @@ +const { Transform } = require('stream') + +/** + * Matches whole blocks as regex and passes them on + */ +class RegexBlockStream extends Transform{ + matcher; + withholdLastBlock; + matchAllOnFlush; + + /** + * @param {RegExp} matcher Block-match - WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more ressources. + * @param {boolean} withholdLastBlock When true, the last matches block will not be submitted to prevent submitting incomplete blocks. + * @param {boolean} matchAllOnFlush (Only in combination with withholdLastBlock) When enabled, the buffer will be matched on last time on _flush (stream deconstruction) and write any, also incomplete, blocks + */ + constructor(matcher, withholdLastBlock = true, matchAllOnFlush = false){ + super({ + readableObjectMode: true, + writableObjectMode: true + }); + + this.matcher = matcher; + this.withholdLastBlock = withholdLastBlock; + this.matchAllOnFlush = matchAllOnFlush; + + this._buffer = ""; + } + + _transform(chunk, encoding, next){ + chunk = this._buffer + chunk; // Add previous buffer to current chunk + + let matches = chunk.match(this.matcher); // Match + if(matches){ + if(this.withholdLastBlock) matches.pop(); // Remove last if we want to withhold it + chunk = this._writeMatches(chunk, matches); + } + + this._buffer = chunk; // Store remaining data in buffer + next(); // Get next chunk + } + + _writeMatches(matches, chunk = null){ + if(matches){ + matches.forEach((match) => { + this.push(match); // Write match to stream + if(chunk) chunk = chunk.replace(match, ''); // Remove match from chunks + }); + } + if(chunk) return chunk; + } + + _flush(next){ + if(matchAllOnFlush){ // When requested, we'll match one last time over the remaining buffer + let matches = this._buffer.match(this.matcher); // Match remaining buffer + _writeMatches(this._buffer); // Write matches including last element + } + + next(); // Tell system we are done + } +} + +// Specify exports +module.exports = { + RegexBlockStream +}; \ No newline at end of file From 4f1463eb4fcfe9313ece54e581584a8d9b5849bd Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 19:40:46 +0100 Subject: [PATCH 017/109] Fixed args being wrong way around --- src/streamHandler/RegexBlockStream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js index f06bf6d..120e65f 100644 --- a/src/streamHandler/RegexBlockStream.js +++ b/src/streamHandler/RegexBlockStream.js @@ -32,7 +32,7 @@ class RegexBlockStream extends Transform{ let matches = chunk.match(this.matcher); // Match if(matches){ if(this.withholdLastBlock) matches.pop(); // Remove last if we want to withhold it - chunk = this._writeMatches(chunk, matches); + chunk = this._writeMatches(matches, chunk); } this._buffer = chunk; // Store remaining data in buffer From 368b6585ba070c2556749c94606ca861d26d0f42 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:00:05 +0100 Subject: [PATCH 018/109] Now using internal buffer-system --- src/streamHandler/RegexBlockStream.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js index 120e65f..62dadfe 100644 --- a/src/streamHandler/RegexBlockStream.js +++ b/src/streamHandler/RegexBlockStream.js @@ -1,3 +1,4 @@ +const logger = require.main.require("./helper/logger.js")("RegexBlockStream"); const { Transform } = require('stream') /** @@ -22,12 +23,11 @@ class RegexBlockStream extends Transform{ this.matcher = matcher; this.withholdLastBlock = withholdLastBlock; this.matchAllOnFlush = matchAllOnFlush; - - this._buffer = ""; } _transform(chunk, encoding, next){ - chunk = this._buffer + chunk; // Add previous buffer to current chunk + chunk = this.readableBuffer.length? this.readableBuffer.join() + chunk: chunk; // Add previous buffer to current chunk + this.readableBuffer.length && this.readableBuffer.clear(); // Clear buffer once we read it let matches = chunk.match(this.matcher); // Match if(matches){ @@ -35,7 +35,7 @@ class RegexBlockStream extends Transform{ chunk = this._writeMatches(matches, chunk); } - this._buffer = chunk; // Store remaining data in buffer + this.readableBuffer.push(chunk); // Store remaining data in buffer next(); // Get next chunk } @@ -51,8 +51,9 @@ class RegexBlockStream extends Transform{ _flush(next){ if(matchAllOnFlush){ // When requested, we'll match one last time over the remaining buffer - let matches = this._buffer.match(this.matcher); // Match remaining buffer - _writeMatches(this._buffer); // Write matches including last element + let chunk = this.readableBuffer.toString(); + let matches = chunk.match(this.matcher); // Match remaining buffer + _writeMatches(matches); // Write matches including last element } next(); // Tell system we are done From 0cbbecbd2c7c8aa768d9be60d6b37257f50a8edf Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:00:28 +0100 Subject: [PATCH 019/109] Added typeMoreData --- src/dto/Packet.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 57a0120..b40997f 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -3,6 +3,7 @@ const PacketType = { ProbeRequest: 'ProbeRequest', ProbeResponse: 'ProbeResponse', Data: 'Data', + MoreData: 'MoreData', RequestToSend: 'RequestToSend', ClearToSend: 'ClearToSend', Acknowledgment: 'Acknowledgment', From 6bc248b667780a3e017ad93e3db84fd3cfdffca0 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:01:03 +0100 Subject: [PATCH 020/109] First implementation of PacketStreamFactory --- src/streamHandler/PacketStreamFactory.js | 96 ++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/streamHandler/PacketStreamFactory.js diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js new file mode 100644 index 0000000..9923f8f --- /dev/null +++ b/src/streamHandler/PacketStreamFactory.js @@ -0,0 +1,96 @@ +const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); +const { Transform } = require('stream'); +const { DateTime } = require("luxon"); +const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket } = require.main.require('./dto/Packet.js'); + +const PACKET_TYPE_MAP = { + "Beacon": PacketType.Beacon, + "Probe Request": PacketType.ProbeRequest, + "Probe Response": PacketType.ProbeResponse, + "Data": PacketType.Data, + "More Data": PacketType.MoreData, + "Request-To-Send": PacketType.RequestToSend, + "Clear-To-Send": PacketType.ClearToSend, + "Acknowledgment": PacketType.Acknowledgment, + "BA": PacketType.BlockAcknowledgment +}; +const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); + +/** + * Read data from text-blocks and convert them to Packet + */ +class PacketStreamFactory extends Transform{ + matcher; + withholdLastBlock; + matchAllOnFlush; + + constructor(){ + super({ + readableObjectMode: true, + writableObjectMode: true + }); + } + + _transform(chunk, encoding, next){ + let packet = new Packet(); + + const lines = chunk.split('\n'); + const header = lines.splice(0, 1)[0]; // Grab first line, 'lines' is now the payload + this._handleHeader(packet, header); + this._handlePayload(packet, lines); + + logger.debug(packet); + + this.read(); + next(null, packet); // Get next chunk + } + + _handleHeader(packet, data){ + // Convert time to epoch-micros Unfortunately luxon doesnt use micros, but millis as smallest time-unit requiring some "hacks" + packet.timestampMicros = DateTime.fromISO(data.slice(0, 12)).toSeconds() + data.slice(12, 15)/1000000; + + packet.dataRate = Number(data.match(/(^| )([0-9]+(\.[0-9]+)?) Mb\/s($| )/i)[2]); + packet.frequency = Number(data.match(/(^| )([0-9]{4}) MHz($| )/i)[2]); + + let signalStrMatch = data.match(/(^| )(-[0-9]{2})dBm Signal($| )/i); + if(signalStrMatch) packet.signal = Number(signalStrMatch[2]); + else packet.signal = -100; + + let packetTypeStrMatch = data.match(new RegExp(`(^|.{80} )(${PACKET_TYPES_REGEX})($| )`, 'i')); + let packetTypeStr; + if(packetTypeStrMatch) { + packetTypeStr = packetTypeStrMatch[2]; + packet.packetType = PACKET_TYPE_MAP[packetTypeStr]; + } + else + packet.packetType = PacketType.Unknown; + + let srcMacMatch = data.match(/(^| )(SA|TA):(.{17})($| )/i); + if(srcMacMatch) packet.srcMac = srcMacMatch[3]; + + let dstMacMatch = data.match(/(^| )(DA|RA):(.{17})($| )/i); + if(dstMacMatch) packet.dstMac = dstMacMatch[3]; + + let bssidMatch = data.match(/(^| )BSSID:(.{17})($| )/i) + if(bssidMatch) packet.bssid = bssidMatch[2]; + + // Cover special cases with more data + switch(packet.packetType){ + case PacketType.Beacon: + case PacketType.ProbeRequest: + case PacketType.ProbeResponse: + packet = Object.assign(new PacketWithSSID(), packet); // Create new, more specific, packet and copy old data over + packet.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))[2]; + break; + } + } + + _handlePayload(packet, data){ + + } +} + +// Specify exports +module.exports = { + PacketStreamFactory +}; \ No newline at end of file From 1aae9d5e719975795811c46d00670a97740d3df7 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:01:33 +0100 Subject: [PATCH 021/109] Added library luxon --- package-lock.json | 16 +++++++++++++++- package.json | 3 ++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 68a64c7..37faa91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "license": "AGPL-3.0", "dependencies": { "@influxdata/influxdb-client": "^1.20.0", - "log4js": "^6.3.0" + "log4js": "^6.3.0", + "luxon": "^2.1.1" } }, "node_modules/@influxdata/influxdb-client": { @@ -88,6 +89,14 @@ "node": ">=8.0" } }, + "node_modules/luxon": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-2.1.1.tgz", + "integrity": "sha512-6VQVNw7+kQu3hL1ZH5GyOhnk8uZm21xS7XJ/6vDZaFNcb62dpFDKcH8TI5NkoZOdMRxr7af7aYGrJlE/Wv0i1w==", + "engines": { + "node": ">=12" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -187,6 +196,11 @@ "streamroller": "^2.2.4" } }, + "luxon": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-2.1.1.tgz", + "integrity": "sha512-6VQVNw7+kQu3hL1ZH5GyOhnk8uZm21xS7XJ/6vDZaFNcb62dpFDKcH8TI5NkoZOdMRxr7af7aYGrJlE/Wv0i1w==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", diff --git a/package.json b/package.json index 7bc22eb..b7b1bd4 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "license": "AGPL-3.0", "dependencies": { "@influxdata/influxdb-client": "^1.20.0", - "log4js": "^6.3.0" + "log4js": "^6.3.0", + "luxon": "^2.1.1" } } From d714b342ed61af4a78fca4a9ea1ceac61f186842 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:35:10 +0100 Subject: [PATCH 022/109] Removed debug read statement --- src/streamHandler/PacketStreamFactory.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 9923f8f..4fc96a6 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -41,7 +41,6 @@ class PacketStreamFactory extends Transform{ logger.debug(packet); - this.read(); next(null, packet); // Get next chunk } From 333a51436376c57e96dab4f956079f2761c62d0f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:35:44 +0100 Subject: [PATCH 023/109] Added PacketType ContentionFreePeriod --- src/dto/Packet.js | 1 + src/streamHandler/PacketStreamFactory.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index b40997f..d411f49 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -9,6 +9,7 @@ const PacketType = { Acknowledgment: 'Acknowledgment', BlockAcknowledgment: 'BlockAcknowledgment', NoData: 'NoData', + ContentionFreePeriod: 'ContentionFreePeriod', Unknown: 'Unknown' } diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 4fc96a6..48e910f 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -12,7 +12,8 @@ const PACKET_TYPE_MAP = { "Request-To-Send": PacketType.RequestToSend, "Clear-To-Send": PacketType.ClearToSend, "Acknowledgment": PacketType.Acknowledgment, - "BA": PacketType.BlockAcknowledgment + "BA": PacketType.BlockAcknowledgment, + "CD": PacketType.ContentionFreePeriod }; const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); From 59e7188543e335583bdb2219cd28312a25ced663 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:39:20 +0100 Subject: [PATCH 024/109] Changed most checks using Optional chaining and Nullish coalescing operator --- src/streamHandler/PacketStreamFactory.js | 29 ++++++++---------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 48e910f..32193b0 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -49,30 +49,19 @@ class PacketStreamFactory extends Transform{ // Convert time to epoch-micros Unfortunately luxon doesnt use micros, but millis as smallest time-unit requiring some "hacks" packet.timestampMicros = DateTime.fromISO(data.slice(0, 12)).toSeconds() + data.slice(12, 15)/1000000; - packet.dataRate = Number(data.match(/(^| )([0-9]+(\.[0-9]+)?) Mb\/s($| )/i)[2]); - packet.frequency = Number(data.match(/(^| )([0-9]{4}) MHz($| )/i)[2]); + packet.dataRate = Number(data.match(/(^| )([0-9]+(\.[0-9]+)?) Mb\/s($| )/i)?.[2]) || null; + packet.frequency = Number(data.match(/(^| )([0-9]{4}) MHz($| )/i)?.[2]) || null; - let signalStrMatch = data.match(/(^| )(-[0-9]{2})dBm Signal($| )/i); - if(signalStrMatch) packet.signal = Number(signalStrMatch[2]); - else packet.signal = -100; + packet.signal = Number(data.match(/(^| )(-[0-9]{2})dBm Signal($| )/i)?.[2]) || null; - let packetTypeStrMatch = data.match(new RegExp(`(^|.{80} )(${PACKET_TYPES_REGEX})($| )`, 'i')); - let packetTypeStr; - if(packetTypeStrMatch) { - packetTypeStr = packetTypeStrMatch[2]; - packet.packetType = PACKET_TYPE_MAP[packetTypeStr]; - } - else - packet.packetType = PacketType.Unknown; + let packetTypeStr = data.match(new RegExp(`(^|.{80} )(${PACKET_TYPES_REGEX})($| )`, 'i'))?.[2]; + packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: PacketType.Unknown; - let srcMacMatch = data.match(/(^| )(SA|TA):(.{17})($| )/i); - if(srcMacMatch) packet.srcMac = srcMacMatch[3]; + packet.srcMac = data.match(/(^| )(SA|TA):(.{17})($| )/i)?.[3] ?? null; - let dstMacMatch = data.match(/(^| )(DA|RA):(.{17})($| )/i); - if(dstMacMatch) packet.dstMac = dstMacMatch[3]; + packet.dstMac = data.match(/(^| )(DA|RA):(.{17})($| )/i)?.[3] ?? null; - let bssidMatch = data.match(/(^| )BSSID:(.{17})($| )/i) - if(bssidMatch) packet.bssid = bssidMatch[2]; + packet.bssid = data.match(/(^| )BSSID:(.{17})($| )/i)?.[2] ?? null; // Cover special cases with more data switch(packet.packetType){ @@ -80,7 +69,7 @@ class PacketStreamFactory extends Transform{ case PacketType.ProbeRequest: case PacketType.ProbeResponse: packet = Object.assign(new PacketWithSSID(), packet); // Create new, more specific, packet and copy old data over - packet.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))[2]; + packet.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))?.[2] ?? null; break; } } From 025edf8465b170c17711a3e2ad62f080409c6183 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:42:10 +0100 Subject: [PATCH 025/109] Implemented getting payloadData and calculating to bytes --- src/streamHandler/PacketStreamFactory.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 32193b0..668683d 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -75,7 +75,9 @@ class PacketStreamFactory extends Transform{ } _handlePayload(packet, data){ - + // Get payload-Hex-Data. If there is no data: empty + let payloadData = data.join('').match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''; + packet.payloadSize = payloadData.length/2; // 2 hex-chars = 1 byte } } From 62b55b94f16cf3c582f5a20f907d5698888925b2 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:42:33 +0100 Subject: [PATCH 026/109] Added missing payloadSize to Packet --- src/dto/Packet.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index d411f49..66b7f80 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -24,6 +24,8 @@ class Packet{ frequency; dataRate; + payloadSize; + packetType; } From aa9d5431f85c1c7a1bb6c03176d9c872dbb8e6a5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 20:44:04 +0100 Subject: [PATCH 027/109] Changed payloadSize to payloadData with payloadSize getter --- src/dto/Packet.js | 5 ++++- src/streamHandler/PacketStreamFactory.js | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 66b7f80..2162779 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -24,7 +24,10 @@ class Packet{ frequency; dataRate; - payloadSize; + payloadData; + get payloadSize(){ + return payloadData.length/2; + } packetType; } diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 668683d..3770075 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -76,8 +76,7 @@ class PacketStreamFactory extends Transform{ _handlePayload(packet, data){ // Get payload-Hex-Data. If there is no data: empty - let payloadData = data.join('').match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''; - packet.payloadSize = payloadData.length/2; // 2 hex-chars = 1 byte + packet.payloadData = data.join('').match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''; } } From 3c3ab18410148d90f6c3672790924afab6c7e1a9 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 21:01:17 +0100 Subject: [PATCH 028/109] Revert "Added PacketType ContentionFreePeriod" This reverts commit 333a51436376c57e96dab4f956079f2761c62d0f. --- src/dto/Packet.js | 1 - src/streamHandler/PacketStreamFactory.js | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 2162779..e908c52 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -9,7 +9,6 @@ const PacketType = { Acknowledgment: 'Acknowledgment', BlockAcknowledgment: 'BlockAcknowledgment', NoData: 'NoData', - ContentionFreePeriod: 'ContentionFreePeriod', Unknown: 'Unknown' } diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 3770075..dffb102 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -12,8 +12,7 @@ const PACKET_TYPE_MAP = { "Request-To-Send": PacketType.RequestToSend, "Clear-To-Send": PacketType.ClearToSend, "Acknowledgment": PacketType.Acknowledgment, - "BA": PacketType.BlockAcknowledgment, - "CD": PacketType.ContentionFreePeriod + "BA": PacketType.BlockAcknowledgment }; const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); From c2210dfb5968bb48b92c3f9630951f179a407d44 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 21:11:19 +0100 Subject: [PATCH 029/109] Added isRetry field --- src/dto/Packet.js | 1 + src/streamHandler/PacketStreamFactory.js | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index e908c52..0b4fda5 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -14,6 +14,7 @@ const PacketType = { class Packet{ timestampMicros; + isRetry; srcMac; dstMac; diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index dffb102..01795da 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -48,6 +48,8 @@ class PacketStreamFactory extends Transform{ // Convert time to epoch-micros Unfortunately luxon doesnt use micros, but millis as smallest time-unit requiring some "hacks" packet.timestampMicros = DateTime.fromISO(data.slice(0, 12)).toSeconds() + data.slice(12, 15)/1000000; + packet.isRetry = data.match(/(^| )Retry($| )/i)? true: false; + packet.dataRate = Number(data.match(/(^| )([0-9]+(\.[0-9]+)?) Mb\/s($| )/i)?.[2]) || null; packet.frequency = Number(data.match(/(^| )([0-9]{4}) MHz($| )/i)?.[2]) || null; From bba5da25998c7cbe5f72ba0a19f181c2bdf7fa4d Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 22:25:10 +0100 Subject: [PATCH 030/109] Moved packet-creation outside for all cases --- src/streamHandler/PacketStreamFactory.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 01795da..0e8f76d 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -65,14 +65,16 @@ class PacketStreamFactory extends Transform{ packet.bssid = data.match(/(^| )BSSID:(.{17})($| )/i)?.[2] ?? null; // Cover special cases with more data + let newPacket; switch(packet.packetType){ case PacketType.Beacon: case PacketType.ProbeRequest: case PacketType.ProbeResponse: - packet = Object.assign(new PacketWithSSID(), packet); // Create new, more specific, packet and copy old data over - packet.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))?.[2] ?? null; + newPacket = new PacketWithSSID(); + newPacket.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))?.[2] ?? null; break; } + if(newPacket) packet = Object.assign(new newPacket, packet); // Use new, more specific, packet and copy old data over } _handlePayload(packet, data){ From 0d472e8cb3182794b8ceade4be21881231dfbbdb Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 22:27:22 +0100 Subject: [PATCH 031/109] Implemented PacketType Authentication --- src/dto/Packet.js | 13 ++++++++++++- src/streamHandler/PacketStreamFactory.js | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 0b4fda5..01d7466 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -9,6 +9,7 @@ const PacketType = { Acknowledgment: 'Acknowledgment', BlockAcknowledgment: 'BlockAcknowledgment', NoData: 'NoData', + Authentication: 'Authentication', Unknown: 'Unknown' } @@ -41,6 +42,14 @@ class BeaconPacket extends PacketWithSSID{} class ProbeRequestPacket extends PacketWithSSID{} class ProbeResponsePacket extends PacketWithSSID{} +const AuthenticationType = { + OpenSystem_1: 'OpenSystem_1', + OpenSystem_2: 'OpenSystem_2', + Unknown: 'Unknown', +} +class AuthenticationPacket extends Packet{ + authenticationType; +} // Specify exports module.exports = { @@ -49,5 +58,7 @@ module.exports = { PacketWithSSID, BeaconPacket, ProbeRequestPacket, - ProbeResponsePacket + ProbeResponsePacket, + AuthenticationType, + AuthenticationPacket, }; diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 0e8f76d..331190e 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -1,7 +1,7 @@ const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); const { Transform } = require('stream'); const { DateTime } = require("luxon"); -const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket } = require.main.require('./dto/Packet.js'); +const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType } = require.main.require('./dto/Packet.js'); const PACKET_TYPE_MAP = { "Beacon": PacketType.Beacon, @@ -12,10 +12,16 @@ const PACKET_TYPE_MAP = { "Request-To-Send": PacketType.RequestToSend, "Clear-To-Send": PacketType.ClearToSend, "Acknowledgment": PacketType.Acknowledgment, - "BA": PacketType.BlockAcknowledgment + "BA": PacketType.BlockAcknowledgment, + "Authentication": PacketType.Authentication, }; const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); +const AUTHENTICATION_TYPE_MAP = { + "(Open System)-1": AuthenticationType.OpenSystem_1, + "(Open System)-2": AuthenticationType.OpenSystem_2, +} + /** * Read data from text-blocks and convert them to Packet */ @@ -73,6 +79,11 @@ class PacketStreamFactory extends Transform{ newPacket = new PacketWithSSID(); newPacket.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))?.[2] ?? null; break; + + case PacketType.Authentication: + newPacket = new AuthenticationPacket(); + newPacket.authenticationType = AUTHENTICATION_TYPE_MAP[data.match(/(?<=(^|\s)Authentication\s).{3,}(?=\:(\s|$))/i)[0]] ?? AuthenticationType.Unknown; + break; } if(newPacket) packet = Object.assign(new newPacket, packet); // Use new, more specific, packet and copy old data over } From fe5dec7860586685262665c79b3517a32465b503 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 22:29:41 +0100 Subject: [PATCH 032/109] Implemented PacketType AssociationRequest & Response --- src/dto/Packet.js | 9 +++++++++ src/streamHandler/PacketStreamFactory.js | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 01d7466..77c8138 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -10,6 +10,8 @@ const PacketType = { BlockAcknowledgment: 'BlockAcknowledgment', NoData: 'NoData', Authentication: 'Authentication', + AssociationRequest: 'AssociationRequest', + AssociationResponse: 'AssociationResponse', Unknown: 'Unknown' } @@ -51,6 +53,11 @@ class AuthenticationPacket extends Packet{ authenticationType; } +class AssociationRequestPacket extends PacketWithSSID{} +class AssociationResponsePacket extends Packet{ + associationIsSuccessful; +} + // Specify exports module.exports = { PacketType, @@ -61,4 +68,6 @@ module.exports = { ProbeResponsePacket, AuthenticationType, AuthenticationPacket, + AssociationRequestPacket, + AssociationResponsePacket, }; diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 331190e..ac25b6a 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -1,7 +1,7 @@ const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); const { Transform } = require('stream'); const { DateTime } = require("luxon"); -const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType } = require.main.require('./dto/Packet.js'); +const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket } = require.main.require('./dto/Packet.js'); const PACKET_TYPE_MAP = { "Beacon": PacketType.Beacon, @@ -14,6 +14,8 @@ const PACKET_TYPE_MAP = { "Acknowledgment": PacketType.Acknowledgment, "BA": PacketType.BlockAcknowledgment, "Authentication": PacketType.Authentication, + "Assoc Request": PacketType.AssociationRequest, + "Assoc Response": PacketType.AssociationResponse, }; const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); @@ -76,6 +78,7 @@ class PacketStreamFactory extends Transform{ case PacketType.Beacon: case PacketType.ProbeRequest: case PacketType.ProbeResponse: + case PacketType.AssociationRequest: newPacket = new PacketWithSSID(); newPacket.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))?.[2] ?? null; break; @@ -84,6 +87,11 @@ class PacketStreamFactory extends Transform{ newPacket = new AuthenticationPacket(); newPacket.authenticationType = AUTHENTICATION_TYPE_MAP[data.match(/(?<=(^|\s)Authentication\s).{3,}(?=\:(\s|$))/i)[0]] ?? AuthenticationType.Unknown; break; + + case PacketType.AssociationResponse: + newPacket = new AssociationResponsePacket(); + newPacket.associationIsSuccessful = data.match(/Assoc Response\s.{0,30}Successful(?=\s|$)/img) ? true : false; + break; } if(newPacket) packet = Object.assign(new newPacket, packet); // Use new, more specific, packet and copy old data over } From d2178f3b737f75f4bd80a7b6102354bda108bb9d Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 22:35:07 +0100 Subject: [PATCH 033/109] Implemented PacketType Disassociation --- src/dto/Packet.js | 5 +++++ src/streamHandler/PacketStreamFactory.js | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 77c8138..3da3015 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -12,6 +12,7 @@ const PacketType = { Authentication: 'Authentication', AssociationRequest: 'AssociationRequest', AssociationResponse: 'AssociationResponse', + Disassociation: 'Disassociation', Unknown: 'Unknown' } @@ -58,6 +59,10 @@ class AssociationResponsePacket extends Packet{ associationIsSuccessful; } +class DisassociationPacket extends Packet{ + disassociationReason; +} + // Specify exports module.exports = { PacketType, diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index ac25b6a..262341f 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -1,7 +1,7 @@ const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); const { Transform } = require('stream'); const { DateTime } = require("luxon"); -const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket } = require.main.require('./dto/Packet.js'); +const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket } = require.main.require('./dto/Packet.js'); const PACKET_TYPE_MAP = { "Beacon": PacketType.Beacon, @@ -16,6 +16,7 @@ const PACKET_TYPE_MAP = { "Authentication": PacketType.Authentication, "Assoc Request": PacketType.AssociationRequest, "Assoc Response": PacketType.AssociationResponse, + "Disassociation:": PacketType.Disassociation, }; const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); @@ -92,6 +93,11 @@ class PacketStreamFactory extends Transform{ newPacket = new AssociationResponsePacket(); newPacket.associationIsSuccessful = data.match(/Assoc Response\s.{0,30}Successful(?=\s|$)/img) ? true : false; break; + + case PacketType.Disassociation: + newPacket = new DisassociationPacket(); + newPacket.disassociationReason = data.match(/(?<=(^|\s)Disassociation:\s).*?(?=\sBSS|$)/img)?.[0] ?? null; + break; } if(newPacket) packet = Object.assign(new newPacket, packet); // Use new, more specific, packet and copy old data over } From e86b5fca9a895aa02917238544ae65bfc34f0f0f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 22:35:59 +0100 Subject: [PATCH 034/109] Added durationMicros --- src/dto/Packet.js | 2 ++ src/streamHandler/PacketStreamFactory.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 3da3015..699dbe0 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -28,6 +28,8 @@ class Packet{ frequency; dataRate; + durationMicros; + payloadData; get payloadSize(){ return payloadData.length/2; diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 262341f..43e4fa5 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -62,6 +62,8 @@ class PacketStreamFactory extends Transform{ packet.dataRate = Number(data.match(/(^| )([0-9]+(\.[0-9]+)?) Mb\/s($| )/i)?.[2]) || null; packet.frequency = Number(data.match(/(^| )([0-9]{4}) MHz($| )/i)?.[2]) || null; + packet.durationMicros = Number(data.match(/(^| )([0-9]{,4})us($| )/i)?.[2] ?? 0); + packet.signal = Number(data.match(/(^| )(-[0-9]{2})dBm Signal($| )/i)?.[2]) || null; let packetTypeStr = data.match(new RegExp(`(^|.{80} )(${PACKET_TYPES_REGEX})($| )`, 'i'))?.[2]; From b32b5cbed7da283074d50e3d6246be8aa255109a Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 23:12:47 +0100 Subject: [PATCH 035/109] Changed regex to use readbehind/ahead instead of groups --- src/streamHandler/PacketStreamFactory.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 43e4fa5..08706ec 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -57,23 +57,23 @@ class PacketStreamFactory extends Transform{ // Convert time to epoch-micros Unfortunately luxon doesnt use micros, but millis as smallest time-unit requiring some "hacks" packet.timestampMicros = DateTime.fromISO(data.slice(0, 12)).toSeconds() + data.slice(12, 15)/1000000; - packet.isRetry = data.match(/(^| )Retry($| )/i)? true: false; + packet.isRetry = data.match(/(?<=^|\s)Retry(?=$|\s)/i)? true: false; - packet.dataRate = Number(data.match(/(^| )([0-9]+(\.[0-9]+)?) Mb\/s($| )/i)?.[2]) || null; - packet.frequency = Number(data.match(/(^| )([0-9]{4}) MHz($| )/i)?.[2]) || null; + packet.dataRate = Number(data.match(/(?<=^|\s)[0-9]+(\.[0-9]+)?(?=\sMb\/?s($|\s))/i)?.[0]) || null; + packet.frequency = Number(data.match(/(?<=^|\s)[0-9]{4}(?=\sMHz($|\s))/i)?.[0]) || null; - packet.durationMicros = Number(data.match(/(^| )([0-9]{,4})us($| )/i)?.[2] ?? 0); + packet.durationMicros = Number(data.match(/(?<=^|\s)[0-9]{1,4}(?=us($|\s))/i)?.[0]) || null; - packet.signal = Number(data.match(/(^| )(-[0-9]{2})dBm Signal($| )/i)?.[2]) || null; + packet.signal = Number(data.match(/(?<=^|\s)-[0-9]{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null; - let packetTypeStr = data.match(new RegExp(`(^|.{80} )(${PACKET_TYPES_REGEX})($| )`, 'i'))?.[2]; + let packetTypeStr = data.match(new RegExp('(?<=^|\\s)('+ PACKET_TYPES_REGEX +')(?=$|\\s)', 'i'))?.[0]; packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: PacketType.Unknown; - packet.srcMac = data.match(/(^| )(SA|TA):(.{17})($| )/i)?.[3] ?? null; + packet.srcMac = data.match(/(?<=(^|\s)(SA|TA):).{17}(?=$|\s)/i)?.[0] ?? null; - packet.dstMac = data.match(/(^| )(DA|RA):(.{17})($| )/i)?.[3] ?? null; + packet.dstMac = data.match(/(?<=(^|\s)(DA|RA):).{17}(?=$|\s)/i)?.[0] ?? null; - packet.bssid = data.match(/(^| )BSSID:(.{17})($| )/i)?.[2] ?? null; + packet.bssid = data.match(/(?<=(^|\s)BSSID:).{17}(?=$|\s)/i)?.[0] ?? null; // Cover special cases with more data let newPacket; @@ -83,7 +83,7 @@ class PacketStreamFactory extends Transform{ case PacketType.ProbeResponse: case PacketType.AssociationRequest: newPacket = new PacketWithSSID(); - newPacket.ssid = data.match(new RegExp(`(^| )${packetTypeStr} `+'\\'+`((.{0,32})`+'\\'+`)($| )`, 'i'))?.[2] ?? null; + newPacket.ssid = data.match(new RegExp('(?<=(^|\\s)'+ packetTypeStr +'\\s\\().{0,32}(?=\\)($|\\s))', 'i'))?.[0] ?? null; break; case PacketType.Authentication: @@ -93,7 +93,7 @@ class PacketStreamFactory extends Transform{ case PacketType.AssociationResponse: newPacket = new AssociationResponsePacket(); - newPacket.associationIsSuccessful = data.match(/Assoc Response\s.{0,30}Successful(?=\s|$)/img) ? true : false; + newPacket.associationIsSuccessful = data.match(/(?<=(^|\s)Assoc\sResponse\s.{0,30})Successful(?=\s|$)/img) ? true : false; break; case PacketType.Disassociation: From 66c0fdfe12b946ea6012d86b7fc43077488371ed Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 23:13:04 +0100 Subject: [PATCH 036/109] Fixed Object.assign --- src/streamHandler/PacketStreamFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 08706ec..d1953db 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -101,7 +101,7 @@ class PacketStreamFactory extends Transform{ newPacket.disassociationReason = data.match(/(?<=(^|\s)Disassociation:\s).*?(?=\sBSS|$)/img)?.[0] ?? null; break; } - if(newPacket) packet = Object.assign(new newPacket, packet); // Use new, more specific, packet and copy old data over + if(newPacket) packet = Object.assign(newPacket, packet); // Use new, more specific, packet and copy old data over } _handlePayload(packet, data){ From 09bea81058515b62f512dcf00257183d56974e28 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 23:13:35 +0100 Subject: [PATCH 037/109] Returning packet not to loose information --- src/streamHandler/PacketStreamFactory.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index d1953db..c99a525 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -45,8 +45,8 @@ class PacketStreamFactory extends Transform{ const lines = chunk.split('\n'); const header = lines.splice(0, 1)[0]; // Grab first line, 'lines' is now the payload - this._handleHeader(packet, header); - this._handlePayload(packet, lines); + packet = this._handleHeader(packet, header); + packet = this._handlePayload(packet, lines); logger.debug(packet); @@ -102,11 +102,15 @@ class PacketStreamFactory extends Transform{ break; } if(newPacket) packet = Object.assign(newPacket, packet); // Use new, more specific, packet and copy old data over + + return packet; } _handlePayload(packet, data){ // Get payload-Hex-Data. If there is no data: empty packet.payloadData = data.join('').match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''; + + return packet; } } From ffe14e3f53e3f9fd2d21b51c14cc6db40cdae622 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Wed, 24 Nov 2021 23:22:09 +0100 Subject: [PATCH 038/109] Added hexConverter helper-module --- src/dto/Packet.js | 14 ++++++++++++++ src/helper/hexConverter.js | 24 ++++++++++++++++++++++++ src/streamHandler/PacketStreamFactory.js | 3 ++- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/helper/hexConverter.js diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 699dbe0..4d1723f 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -13,6 +13,7 @@ const PacketType = { AssociationRequest: 'AssociationRequest', AssociationResponse: 'AssociationResponse', Disassociation: 'Disassociation', + Handshake: 'Handshake', Unknown: 'Unknown' } @@ -65,6 +66,17 @@ class DisassociationPacket extends Packet{ disassociationReason; } + +const HandshakeStage = { + 1: '1', + 2: '2', + 3: '3', + 4: '4' +} +class HandshakePacket extends Packet{ + handshakeStage; +} + // Specify exports module.exports = { PacketType, @@ -77,4 +89,6 @@ module.exports = { AuthenticationPacket, AssociationRequestPacket, AssociationResponsePacket, + HandshakeStage, + HandshakePacket, }; diff --git a/src/helper/hexConverter.js b/src/helper/hexConverter.js new file mode 100644 index 0000000..00c2e8c --- /dev/null +++ b/src/helper/hexConverter.js @@ -0,0 +1,24 @@ +// From https://stackoverflow.com/a/34356351 + +// Convert a hex string to a byte array +function hexToBytes(hex) { + for (var bytes = [], c = 0; c < hex.length; c += 2) + bytes.push(parseInt(hex.substr(c, 2), 16)); + return bytes; +} + +// Convert a byte array to a hex string +function bytesToHex(bytes) { + for (var hex = [], i = 0; i < bytes.length; i++) { + var current = bytes[i] < 0 ? bytes[i] + 256 : bytes[i]; + hex.push((current >>> 4).toString(16)); + hex.push((current & 0xF).toString(16)); + } + return hex.join(""); +} + +// Specify exports +module.exports = { + hexToBytes, + bytesToHex +} \ No newline at end of file diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index c99a525..0a5b980 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -1,7 +1,7 @@ const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); const { Transform } = require('stream'); const { DateTime } = require("luxon"); -const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket } = require.main.require('./dto/Packet.js'); +const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket, HandshakePacket, HandshakeStage } = require.main.require('./dto/Packet.js'); const PACKET_TYPE_MAP = { "Beacon": PacketType.Beacon, @@ -17,6 +17,7 @@ const PACKET_TYPE_MAP = { "Assoc Request": PacketType.AssociationRequest, "Assoc Response": PacketType.AssociationResponse, "Disassociation:": PacketType.Disassociation, + "EAPOL": PacketType.Handshake, }; const PACKET_TYPES_REGEX = Object.keys(PACKET_TYPE_MAP).join('|'); From d6135a02e9713e887b8a14f3131b50eb03139be5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 00:18:21 +0100 Subject: [PATCH 039/109] Fixed regex --- src/streamHandler/PacketStreamFactory.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 0a5b980..1819e69 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -94,12 +94,12 @@ class PacketStreamFactory extends Transform{ case PacketType.AssociationResponse: newPacket = new AssociationResponsePacket(); - newPacket.associationIsSuccessful = data.match(/(?<=(^|\s)Assoc\sResponse\s.{0,30})Successful(?=\s|$)/img) ? true : false; + newPacket.associationIsSuccessful = data.match(/(?<=(^|\s)Assoc\sResponse\s.{0,30})Successful(?=\s|$)/i) ? true : false; break; case PacketType.Disassociation: newPacket = new DisassociationPacket(); - newPacket.disassociationReason = data.match(/(?<=(^|\s)Disassociation:\s).*?(?=\sBSS|$)/img)?.[0] ?? null; + newPacket.disassociationReason = data.match(/(?<=(^|\s)Disassociation:\s).*$/i)?.[0] ?? null; break; } if(newPacket) packet = Object.assign(newPacket, packet); // Use new, more specific, packet and copy old data over From b41d728ba9b5cb82382a767b4028b32c9c2775aa Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 00:18:37 +0100 Subject: [PATCH 040/109] Fixed missing export --- src/dto/Packet.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 4d1723f..dd1149b 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -89,6 +89,7 @@ module.exports = { AuthenticationPacket, AssociationRequestPacket, AssociationResponsePacket, + DisassociationPacket, HandshakeStage, HandshakePacket, }; From 57394882cc6a1b24f44e29ebf8441452769c8e9b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 00:19:31 +0100 Subject: [PATCH 041/109] Implemented reading keyExchangeData and determining handshake-number --- src/streamHandler/PacketStreamFactory.js | 36 +++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 1819e69..59e29c2 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -2,6 +2,7 @@ const logger = require.main.require("./helper/logger.js")("PacketStreamFactory") const { Transform } = require('stream'); const { DateTime } = require("luxon"); const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket, HandshakePacket, HandshakeStage } = require.main.require('./dto/Packet.js'); +const hexConv = require.main.require("./helper/hexConverter.js"); const PACKET_TYPE_MAP = { "Beacon": PacketType.Beacon, @@ -108,8 +109,41 @@ class PacketStreamFactory extends Transform{ } _handlePayload(packet, data){ + data = data.join(''); + // Get payload-Hex-Data. If there is no data: empty - packet.payloadData = data.join('').match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''; + packet.payloadData = hexConv.hexToBytes(data.match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''); + + // Cover special cases with more data + let newPacket; + switch(packet.packetType){ + case PacketType.Handshake: + newPacket = new HandshakePacket(); + + // Read key-information + let keyInfoRaw = (packet.payloadData[0x5]<<0x8) + packet.payloadData[0x6]; + let keyInfo = { + "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111, + "KeyType": keyInfoRaw>>3 & 0b1, + "KeyIndex": keyInfoRaw>>4 & 0b11, + "Install": keyInfoRaw>>6 & 0b1, + "KeyACK": keyInfoRaw>>7 & 0b1, + "KeyMIC": keyInfoRaw>>8 & 0b1, + "Secure": keyInfoRaw>>9 & 0b1, + "Error": keyInfoRaw>>10 & 0b1, + "Request": keyInfoRaw>>11 & 0b1, + "EncryptedKeyData": keyInfoRaw>>12 & 0b1, + "SMKMessage": keyInfoRaw>>13 & 0b1, + }; + + newPacket.handshakeStage = (!keyInfo.Install && keyInfo.KeyACK && !keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[1] : + (!keyInfo.Install && !keyInfo.KeyACK && keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[2] : + ( keyInfo.Install && keyInfo.KeyACK && keyInfo.KeyMIC && keyInfo.Secure)? HandshakeStage[3] : + (!keyInfo.Install && !keyInfo.KeyACK && keyInfo.KeyMIC && keyInfo.Secure)? HandshakeStage[4] : + null; + break; + } + if(newPacket) packet = Object.assign(newPacket, packet); return packet; } From d0ee1a70441d3c1ff60cecbdceb15841454fe5f5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 00:19:53 +0100 Subject: [PATCH 042/109] Fixed gtter payloadSize --- src/dto/Packet.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index dd1149b..452290b 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -33,7 +33,7 @@ class Packet{ payloadData; get payloadSize(){ - return payloadData.length/2; + return this.payloadData.length; } packetType; From 7bc5a3530c65f0e61b355b11c43eda91d417e12b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 00:21:25 +0100 Subject: [PATCH 043/109] Made keyInfo vars const --- src/streamHandler/PacketStreamFactory.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 59e29c2..ca98642 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -121,8 +121,8 @@ class PacketStreamFactory extends Transform{ newPacket = new HandshakePacket(); // Read key-information - let keyInfoRaw = (packet.payloadData[0x5]<<0x8) + packet.payloadData[0x6]; - let keyInfo = { + const keyInfoRaw = (packet.payloadData[0x5]<<0x8) + packet.payloadData[0x6]; + const keyInfo = { "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111, "KeyType": keyInfoRaw>>3 & 0b1, "KeyIndex": keyInfoRaw>>4 & 0b11, From 68541b1191259f39e4cd6bc3cbeeae9092b327f5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 01:09:25 +0100 Subject: [PATCH 044/109] Added check for NoData-Packet --- src/streamHandler/PacketStreamFactory.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index ca98642..8113677 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -69,7 +69,9 @@ class PacketStreamFactory extends Transform{ packet.signal = Number(data.match(/(?<=^|\s)-[0-9]{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null; let packetTypeStr = data.match(new RegExp('(?<=^|\\s)('+ PACKET_TYPES_REGEX +')(?=$|\\s)', 'i'))?.[0]; - packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: PacketType.Unknown; + packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: + data.match(/(SA|TA|DA|RA|BSSID):.{17}\s*$/i)? PacketType.NoData: + PacketType.Unknown; packet.srcMac = data.match(/(?<=(^|\s)(SA|TA):).{17}(?=$|\s)/i)?.[0] ?? null; From 25297ed4d982dbc5659f5855ffca9252f1f4b509 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 01:10:34 +0100 Subject: [PATCH 045/109] Fixed data-regex --- src/streamHandler/PacketStreamFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 8113677..ee66562 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -114,7 +114,7 @@ class PacketStreamFactory extends Transform{ data = data.join(''); // Get payload-Hex-Data. If there is no data: empty - packet.payloadData = hexConv.hexToBytes(data.match(/(?<=\s)([A-F0-9]{1,4}(?!(\.|x)))/igm)?.join('') ?? ''); + packet.payloadData = hexConv.hexToBytes(data.match(/(?<=\s)([A-F0-9]{1,4}(?=\s))/igm)?.join('') ?? ''); // Cover special cases with more data let newPacket; From 21e4ff1a669dbdebe943ae649403574db62dd7c5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 01:10:49 +0100 Subject: [PATCH 046/109] Removing frameCheckSequence from packets --- src/streamHandler/PacketStreamFactory.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index ee66562..2d64183 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -115,6 +115,7 @@ class PacketStreamFactory extends Transform{ // Get payload-Hex-Data. If there is no data: empty packet.payloadData = hexConv.hexToBytes(data.match(/(?<=\s)([A-F0-9]{1,4}(?=\s))/igm)?.join('') ?? ''); + packet.payloadData.splice(packet.payloadData.length-4, 4); // Remove FrameCheck sequence // Cover special cases with more data let newPacket; From a1ce7a848bf28793a6c1efa75845516fa8f4e36c Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 01:37:35 +0100 Subject: [PATCH 047/109] Replaced MoreData and isRetry with flags --- src/dto/Packet.js | 14 ++++++++++++-- src/streamHandler/PacketStreamFactory.js | 18 ++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/dto/Packet.js b/src/dto/Packet.js index 452290b..712daea 100644 --- a/src/dto/Packet.js +++ b/src/dto/Packet.js @@ -3,7 +3,6 @@ const PacketType = { ProbeRequest: 'ProbeRequest', ProbeResponse: 'ProbeResponse', Data: 'Data', - MoreData: 'MoreData', RequestToSend: 'RequestToSend', ClearToSend: 'ClearToSend', Acknowledgment: 'Acknowledgment', @@ -17,9 +16,19 @@ const PacketType = { Unknown: 'Unknown' } +const FlagType = { + MoreFragments: "MoreFragments", + Retry: "Retry", + PwrMgt: "PwrMgt", + MoreData: "MoreData", + Protected: "Protected", + Order: "Order" +} + class Packet{ timestampMicros; - isRetry; + + flags = {}; srcMac; dstMac; @@ -80,6 +89,7 @@ class HandshakePacket extends Packet{ // Specify exports module.exports = { PacketType, + FlagType, Packet, PacketWithSSID, BeaconPacket, diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 2d64183..9bc9b7b 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -1,7 +1,7 @@ const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); const { Transform } = require('stream'); const { DateTime } = require("luxon"); -const { PacketType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket, HandshakePacket, HandshakeStage } = require.main.require('./dto/Packet.js'); +const { PacketType, FlagType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket, HandshakePacket, HandshakeStage } = require.main.require('./dto/Packet.js'); const hexConv = require.main.require("./helper/hexConverter.js"); const PACKET_TYPE_MAP = { @@ -9,7 +9,6 @@ const PACKET_TYPE_MAP = { "Probe Request": PacketType.ProbeRequest, "Probe Response": PacketType.ProbeResponse, "Data": PacketType.Data, - "More Data": PacketType.MoreData, "Request-To-Send": PacketType.RequestToSend, "Clear-To-Send": PacketType.ClearToSend, "Acknowledgment": PacketType.Acknowledgment, @@ -27,6 +26,14 @@ const AUTHENTICATION_TYPE_MAP = { "(Open System)-2": AuthenticationType.OpenSystem_2, } +const FLAG_TYPE_MAP = { + "Retry": FlagType.Retry, + "Pwr Mgmt": FlagType.PwrMgt, + "More Data": FlagType.MoreData, + "Protected": FlagType.Protected, +} +const FLAG_TYPE_MAPS_REGEX = Object.keys(FLAG_TYPE_MAP).join('|'); + /** * Read data from text-blocks and convert them to Packet */ @@ -59,7 +66,10 @@ class PacketStreamFactory extends Transform{ // Convert time to epoch-micros Unfortunately luxon doesnt use micros, but millis as smallest time-unit requiring some "hacks" packet.timestampMicros = DateTime.fromISO(data.slice(0, 12)).toSeconds() + data.slice(12, 15)/1000000; - packet.isRetry = data.match(/(?<=^|\s)Retry(?=$|\s)/i)? true: false; + // Find flags + data.match(data.match(new RegExp('(?<=^|\\s)('+ FLAG_TYPE_MAPS_REGEX +')(?=$|\\s)', 'ig')) + ?.forEach(match => packet.flags[FLAG_TYPE_MAP[match]] = true) // Set them to true in flags + ); packet.dataRate = Number(data.match(/(?<=^|\s)[0-9]+(\.[0-9]+)?(?=\sMb\/?s($|\s))/i)?.[0]) || null; packet.frequency = Number(data.match(/(?<=^|\s)[0-9]{4}(?=\sMHz($|\s))/i)?.[0]) || null; @@ -72,7 +82,7 @@ class PacketStreamFactory extends Transform{ packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: data.match(/(SA|TA|DA|RA|BSSID):.{17}\s*$/i)? PacketType.NoData: PacketType.Unknown; - + packet.srcMac = data.match(/(?<=(^|\s)(SA|TA):).{17}(?=$|\s)/i)?.[0] ?? null; packet.dstMac = data.match(/(?<=(^|\s)(DA|RA):).{17}(?=$|\s)/i)?.[0] ?? null; From 1976838d8abc68324e667244eb7e998432157f31 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 01:48:07 +0100 Subject: [PATCH 048/109] Remove unecessary properties --- src/streamHandler/PacketStreamFactory.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 9bc9b7b..e64ae03 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -38,10 +38,6 @@ const FLAG_TYPE_MAPS_REGEX = Object.keys(FLAG_TYPE_MAP).join('|'); * Read data from text-blocks and convert them to Packet */ class PacketStreamFactory extends Transform{ - matcher; - withholdLastBlock; - matchAllOnFlush; - constructor(){ super({ readableObjectMode: true, From 38985ea9e298f2e240eb4f31154d09b455835813 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 01:48:41 +0100 Subject: [PATCH 049/109] Remove debug-logging for now --- src/streamHandler/PacketStreamFactory.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index e64ae03..ea1f5cb 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -53,8 +53,6 @@ class PacketStreamFactory extends Transform{ packet = this._handleHeader(packet, header); packet = this._handlePayload(packet, lines); - logger.debug(packet); - next(null, packet); // Get next chunk } From 7ebcf573b96234dc672a7b6052f3317a6e970448 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 02:54:43 +0100 Subject: [PATCH 050/109] Implemented stream to transform packets to datapoints --- src/streamHandler/PacketInfluxPointFactory.js | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/streamHandler/PacketInfluxPointFactory.js diff --git a/src/streamHandler/PacketInfluxPointFactory.js b/src/streamHandler/PacketInfluxPointFactory.js new file mode 100644 index 0000000..11e502f --- /dev/null +++ b/src/streamHandler/PacketInfluxPointFactory.js @@ -0,0 +1,72 @@ +const logger = require.main.require("./helper/logger.js")("PacketStreamFactory"); +const { Transform } = require('stream'); +const {Point} = require('@influxdata/influxdb-client') + +/** Keys to always use as tags */ +const TAG_LIST = [ + "srcMac", + "dstMac", + "bssid", + "frequency", + "flags", +]; + +/** Measurement-name and corresponding field-key */ +const MEASUREMENT_MAP = new Map([ + ["Signal", "signal"], + ["PayloadSize", "payloadSize"], + ["DataRate", "dataRate"], + ["SSID", "ssid"], + ["AuthenticationType", "authenticationType"], + ["AssociationSuccess", "associationIsSuccessful"], + ["DisassociationReason", "disassociationReason"], +]); + + +/** + * Get packets and convert them into influx-points + */ +class PacketInfluxPointFactory extends Transform{ + constructor(){ + super({ + readableObjectMode: true, + writableObjectMode: true + }); + } + + _transform(packet, encoding, next){ + // Create measurements + MEASUREMENT_MAP.forEach((objKey, measurement) => { + if(!Object.keys(packet).includes(objKey)) return; + + let point = new Point(measurement); // Create point + + // Set tags + TAG_LIST.filter(tag => Object.keys(packet).includes(tag)) + .forEach(tag => point.tag(tag, packet[tag])); + + point.setField('value', packet[objKey]); // Set field + + this.push(null, point); // Push point into stream + }); + + next(); // Get next packet + } +} + + +/** Mapping for type -> field-method */ +const POINT_FIELD_TYPE = new Map([ + ['boolean', function(key, value){ return this.booleanField(key, value); }], + ['number', function(key, value){ return this.intField(key, value); }], + ['string', function(key, value){ return this.stringField(key, value); }], +]); +Point.prototype.setField = function(key, value){ + let setField = POINT_FIELD_TYPE.get(typeof value); + return setField.apply(this, [key, value]); +} + +// Specify exports +module.exports = { + PacketInfluxPointFactory +}; \ No newline at end of file From 450f162cda0ebce6aa9e03d51400039ec1dd5b2e Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 03:13:05 +0100 Subject: [PATCH 051/109] Implemented InfluxPointWriter for writing into influx with stream --- src/streamHandler/InfluxPointWriter.js | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/streamHandler/InfluxPointWriter.js diff --git a/src/streamHandler/InfluxPointWriter.js b/src/streamHandler/InfluxPointWriter.js new file mode 100644 index 0000000..808e8d6 --- /dev/null +++ b/src/streamHandler/InfluxPointWriter.js @@ -0,0 +1,38 @@ +const logger = require.main.require("./helper/logger.js")("InfluxPointWriter"); +const { Writeable } = require('stream'); +const {InfluxDB, Point, HttpError} = require('@influxdata/influxdb-client') + +/** + * Get points and write them into influx + */ +class InfluxPointWriter extends Writeable{ + /** + * + * @param {string} url Influx-Url + * @param {string} token Auth-token + * @param {string} org Organization to use + * @param {string} bucket Bucket to use + * @param {string} precision Precision to use + */ + constructor(url, token, org, bucket, precision = 'us'){ + this._api = new InfluxDB({url, token}).getWriteApi(org, bucket, precision); + } + + _write(point, encoding, next){ + this._api.writePoint(point); + next(); + } + + _flush(next){ + this._api.flush(true) + .then( + next, + (err) => { next(new Error(`WriteApi rejected promise for flush: ${err}`)); } + ); + } +} + +// Specify exports +module.exports = { + InfluxPointWriter +}; \ No newline at end of file From cf1b300f6a375bff6e4a865034a91ee02c2e6261 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 03:14:24 +0100 Subject: [PATCH 052/109] Changed documentation to remarks section --- src/streamHandler/RegexBlockStream.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js index 62dadfe..8a713c4 100644 --- a/src/streamHandler/RegexBlockStream.js +++ b/src/streamHandler/RegexBlockStream.js @@ -10,9 +10,10 @@ class RegexBlockStream extends Transform{ matchAllOnFlush; /** - * @param {RegExp} matcher Block-match - WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more ressources. + * @param {RegExp} matcher Block-match * @param {boolean} withholdLastBlock When true, the last matches block will not be submitted to prevent submitting incomplete blocks. * @param {boolean} matchAllOnFlush (Only in combination with withholdLastBlock) When enabled, the buffer will be matched on last time on _flush (stream deconstruction) and write any, also incomplete, blocks + * @remarks WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more ressources. */ constructor(matcher, withholdLastBlock = true, matchAllOnFlush = false){ super({ From e715cc1cac588122411023dbb17bdf68ecc7dc36 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 13:59:45 +0100 Subject: [PATCH 053/109] Removed unused env-vars for now --- src/main.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.js b/src/main.js index c1569b7..aa95a1c 100644 --- a/src/main.js +++ b/src/main.js @@ -9,8 +9,6 @@ const env = process.env; { env.LOGLEVEL ??= "INFO"; env.WIFI_INTERFACE ??= "wlan0"; - env.WIFI_CHANNEL ??= [1,6,11]; - env.WIFI_CHANNEL_TIME ??= 1; } // Required vars let errorMsg = requireEnvVars([ From d77e3f88442a49dcc00f1748d360441a37f34c9e Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 18:30:50 +0100 Subject: [PATCH 054/109] Fixed naming --- src/streamHandler/InfluxPointWriter.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/streamHandler/InfluxPointWriter.js b/src/streamHandler/InfluxPointWriter.js index 808e8d6..02b0a27 100644 --- a/src/streamHandler/InfluxPointWriter.js +++ b/src/streamHandler/InfluxPointWriter.js @@ -1,11 +1,11 @@ const logger = require.main.require("./helper/logger.js")("InfluxPointWriter"); -const { Writeable } = require('stream'); +const { Writable } = require('stream'); const {InfluxDB, Point, HttpError} = require('@influxdata/influxdb-client') /** * Get points and write them into influx */ -class InfluxPointWriter extends Writeable{ +class InfluxPointWriter extends Writable{ /** * * @param {string} url Influx-Url From 1a9ced0bb8021369c25c3de238abbd8c601e249c Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 18:32:07 +0100 Subject: [PATCH 055/109] Changed exec to only wrap spawn --- src/helper/exec.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/helper/exec.js b/src/helper/exec.js index 63f6c69..5523f5c 100644 --- a/src/helper/exec.js +++ b/src/helper/exec.js @@ -3,18 +3,11 @@ const logger = require("./logger.js")("exec"); const { spawn } = require("child_process"); -function exec(cmd, stdout, stderr, exit_handler){ - const [bin, ...args] = cmd.split(' ') +function exec(cmd, options){ logger.addContext("binary", "bin"); logger.debug(`Spawn process '${cmd}'`); - let proc = spawn(bin, args); - - return { - "process": proc, - "stdout": proc.stdout, - "stderr": proc.stderr - } + return spawn(bin, args, options); } // Specify exports From dcd0ce8111613025aa3ae9152a62e7c6cff5d1c6 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 18:32:43 +0100 Subject: [PATCH 056/109] Added lib string-argv to extract bin and args from cmd --- package-lock.json | 16 +++++++++++++++- package.json | 3 ++- src/helper/exec.js | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 37faa91..e80026a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "dependencies": { "@influxdata/influxdb-client": "^1.20.0", "log4js": "^6.3.0", - "luxon": "^2.1.1" + "luxon": "^2.1.1", + "string-argv": "^0.3.1" } }, "node_modules/@influxdata/influxdb-client": { @@ -128,6 +129,14 @@ "node": ">=4.0" } }, + "node_modules/string-argv": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz", + "integrity": "sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==", + "engines": { + "node": ">=0.6.19" + } + }, "node_modules/universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -228,6 +237,11 @@ } } }, + "string-argv": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz", + "integrity": "sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==" + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", diff --git a/package.json b/package.json index b7b1bd4..233ef01 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "dependencies": { "@influxdata/influxdb-client": "^1.20.0", "log4js": "^6.3.0", - "luxon": "^2.1.1" + "luxon": "^2.1.1", + "string-argv": "^0.3.1" } } diff --git a/src/helper/exec.js b/src/helper/exec.js index 5523f5c..ed7f628 100644 --- a/src/helper/exec.js +++ b/src/helper/exec.js @@ -1,9 +1,11 @@ const logger = require("./logger.js")("exec"); const { spawn } = require("child_process"); +const { parseArgsStringToArgv } = require('string-argv'); function exec(cmd, options){ + const [bin, ...args] = parseArgsStringToArgv(cmd); logger.addContext("binary", "bin"); logger.debug(`Spawn process '${cmd}'`); From 9472ed9198b15cc4a1063238d64440da98439745 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 18:42:26 +0100 Subject: [PATCH 057/109] Fix wrong usage of join() Default splitter is ',' but we want nothing --- src/streamHandler/RegexBlockStream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js index 8a713c4..5868319 100644 --- a/src/streamHandler/RegexBlockStream.js +++ b/src/streamHandler/RegexBlockStream.js @@ -27,7 +27,7 @@ class RegexBlockStream extends Transform{ } _transform(chunk, encoding, next){ - chunk = this.readableBuffer.length? this.readableBuffer.join() + chunk: chunk; // Add previous buffer to current chunk + chunk = this.readableBuffer.length? this.readableBuffer.join('') + chunk: chunk; // Add previous buffer to current chunk this.readableBuffer.length && this.readableBuffer.clear(); // Clear buffer once we read it let matches = chunk.match(this.matcher); // Match From bb3d843895a44748aea043817598279cfce2abde Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 25 Nov 2021 18:42:45 +0100 Subject: [PATCH 058/109] Fixed wrong joining --- src/streamHandler/RegexBlockStream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js index 5868319..ef39a67 100644 --- a/src/streamHandler/RegexBlockStream.js +++ b/src/streamHandler/RegexBlockStream.js @@ -52,7 +52,7 @@ class RegexBlockStream extends Transform{ _flush(next){ if(matchAllOnFlush){ // When requested, we'll match one last time over the remaining buffer - let chunk = this.readableBuffer.toString(); + let chunk = this.readableBuffer.join(''); let matches = chunk.match(this.matcher); // Match remaining buffer _writeMatches(matches); // Write matches including last element } From 7de2250983489ab4702e4a392946e637d17578f0 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:30:46 +0100 Subject: [PATCH 059/109] Added client-apis --- package-lock.json | 15 +++++++++++++++ package.json | 1 + 2 files changed, 16 insertions(+) diff --git a/package-lock.json b/package-lock.json index e80026a..4e83226 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "AGPL-3.0", "dependencies": { "@influxdata/influxdb-client": "^1.20.0", + "@influxdata/influxdb-client-apis": "^1.20.0", "log4js": "^6.3.0", "luxon": "^2.1.1", "string-argv": "^0.3.1" @@ -20,6 +21,14 @@ "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.20.0.tgz", "integrity": "sha512-jaKSI63hmQ5VSkJrFJkYIXaKlhoF+mGd4HmOf7v/X7pmEi69ReHp922Wyx6/OeCrpndRMbsadk+XmGNdd43cFw==" }, + "node_modules/@influxdata/influxdb-client-apis": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client-apis/-/influxdb-client-apis-1.20.0.tgz", + "integrity": "sha512-KMTmXH4rbpS+NWGpqDjxcKTyan2rbiT2IM5AdRElKhH2sHbH96xwLgziaxeC+OCJLeNAdehJgae3I8WiZjbwdg==", + "peerDependencies": { + "@influxdata/influxdb-client": "*" + } + }, "node_modules/date-format": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/date-format/-/date-format-3.0.0.tgz", @@ -152,6 +161,12 @@ "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.20.0.tgz", "integrity": "sha512-jaKSI63hmQ5VSkJrFJkYIXaKlhoF+mGd4HmOf7v/X7pmEi69ReHp922Wyx6/OeCrpndRMbsadk+XmGNdd43cFw==" }, + "@influxdata/influxdb-client-apis": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client-apis/-/influxdb-client-apis-1.20.0.tgz", + "integrity": "sha512-KMTmXH4rbpS+NWGpqDjxcKTyan2rbiT2IM5AdRElKhH2sHbH96xwLgziaxeC+OCJLeNAdehJgae3I8WiZjbwdg==", + "requires": {} + }, "date-format": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/date-format/-/date-format-3.0.0.tgz", diff --git a/package.json b/package.json index 233ef01..4f3a8dd 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "license": "AGPL-3.0", "dependencies": { "@influxdata/influxdb-client": "^1.20.0", + "@influxdata/influxdb-client-apis": "^1.20.0", "log4js": "^6.3.0", "luxon": "^2.1.1", "string-argv": "^0.3.1" From d7a9530b68e924a26a7f55bf9bed54ea4260a2e7 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:31:31 +0100 Subject: [PATCH 060/109] Cleaned up Promise --- src/streamHandler/InfluxPointWriter.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/streamHandler/InfluxPointWriter.js b/src/streamHandler/InfluxPointWriter.js index 02b0a27..9a57136 100644 --- a/src/streamHandler/InfluxPointWriter.js +++ b/src/streamHandler/InfluxPointWriter.js @@ -25,10 +25,8 @@ class InfluxPointWriter extends Writable{ _flush(next){ this._api.flush(true) - .then( - next, - (err) => { next(new Error(`WriteApi rejected promise for flush: ${err}`)); } - ); + .catch((err) => { next(new Error(`WriteApi rejected promise for flush: ${err}`)); }) + .then(next); } } From 2a662e0bd1f7c594d031bbb8e5906f69d12e880f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:32:31 +0100 Subject: [PATCH 061/109] Changed constructor to take influxDb --- src/streamHandler/InfluxPointWriter.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/streamHandler/InfluxPointWriter.js b/src/streamHandler/InfluxPointWriter.js index 9a57136..fbf64cc 100644 --- a/src/streamHandler/InfluxPointWriter.js +++ b/src/streamHandler/InfluxPointWriter.js @@ -8,14 +8,12 @@ const {InfluxDB, Point, HttpError} = require('@influxdata/influxdb-client') class InfluxPointWriter extends Writable{ /** * - * @param {string} url Influx-Url - * @param {string} token Auth-token + * @param {InfluxDB} influxDb InfluxDb * @param {string} org Organization to use * @param {string} bucket Bucket to use - * @param {string} precision Precision to use + * @param {Partial} options Options for WriteApi */ - constructor(url, token, org, bucket, precision = 'us'){ - this._api = new InfluxDB({url, token}).getWriteApi(org, bucket, precision); + this._api = influxDb.getWriteApi(org, bucket, 'us', options); } _write(point, encoding, next){ From 3af4bb7cc689ea20685736f04f1bcbfe7f2bcaf5 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:33:07 +0100 Subject: [PATCH 062/109] Fixed wrong push --- src/streamHandler/PacketInfluxPointFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/PacketInfluxPointFactory.js b/src/streamHandler/PacketInfluxPointFactory.js index 11e502f..8d76878 100644 --- a/src/streamHandler/PacketInfluxPointFactory.js +++ b/src/streamHandler/PacketInfluxPointFactory.js @@ -47,7 +47,7 @@ class PacketInfluxPointFactory extends Transform{ point.setField('value', packet[objKey]); // Set field - this.push(null, point); // Push point into stream + this.push(point); // Push point into stream }); next(); // Get next packet From 44cd3288cf95ef862de8c22f2c909e0b8b3ca721 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:33:19 +0100 Subject: [PATCH 063/109] Fixed stram not being in object-mode --- src/streamHandler/InfluxPointWriter.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/streamHandler/InfluxPointWriter.js b/src/streamHandler/InfluxPointWriter.js index fbf64cc..9bd96ad 100644 --- a/src/streamHandler/InfluxPointWriter.js +++ b/src/streamHandler/InfluxPointWriter.js @@ -13,6 +13,10 @@ class InfluxPointWriter extends Writable{ * @param {string} bucket Bucket to use * @param {Partial} options Options for WriteApi */ + constructor(influxDb, org, bucket, options){ + super({ + objectMode: true + }); this._api = influxDb.getWriteApi(org, bucket, 'us', options); } From d10e9bb2c67dd4b3c1b0c55185722fc79c2ae8fb Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:43:42 +0100 Subject: [PATCH 064/109] Create influx-client --- src/main.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.js b/src/main.js index aa95a1c..95e1037 100644 --- a/src/main.js +++ b/src/main.js @@ -2,6 +2,7 @@ const logger = require("./helper/logger.js")("main"); const { requireEnvVars } = require("./helper/env.js"); const { exit } = require("process"); +const { InfluxDB } = require('@influxdata/influxdb-client'); /// Setup ENVs const env = process.env; @@ -20,3 +21,8 @@ if(errorMsg){ exit(1); } +(async function() { + logger.info("Setup Influx.."); + const influxDb = new InfluxDB({url: env.INFLUX_URL, token: env.INFLUX_TOKEN}); + +})(); From fc5900b0baeb21a16f935d48abe15fbd761d12dc Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 17:55:51 +0100 Subject: [PATCH 065/109] Added influx checkHealth --- src/helper/influx-checks.js | 26 ++++++++++++++++++++++++++ src/main.js | 4 ++++ 2 files changed, 30 insertions(+) create mode 100644 src/helper/influx-checks.js diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js new file mode 100644 index 0000000..b44a0fc --- /dev/null +++ b/src/helper/influx-checks.js @@ -0,0 +1,26 @@ +const logger = require("./helper/logger.js")("influx-checks"); + +const Influx = require('@influxdata/influxdb-client-apis'); + + +function checkHealth(influxDb){ + return new Promise((resolve, reject) => { + new Influx.HealthAPI(influxDb) // Check influx health + .getHealth() + .catch((err) => { + logger.fatal("Could not communicate with Influx:"); + logger.fatal(`Error [${err.code}]:`, err.message); + reject(err); + }) + .then((res) => { + logger.debug("Server healthy.", "Version: ", res.version); + resolve(res); + }); + }); +} + + +// Specify exports +module.exports = { + checkHealth, +}; \ No newline at end of file diff --git a/src/main.js b/src/main.js index 95e1037..dec1e82 100644 --- a/src/main.js +++ b/src/main.js @@ -25,4 +25,8 @@ if(errorMsg){ logger.info("Setup Influx.."); const influxDb = new InfluxDB({url: env.INFLUX_URL, token: env.INFLUX_TOKEN}); + await InfluxChecks.checkHealth(influxDb) + .catch(exit(1)) + .then((res) => {}); + })(); From e1b2a7e016f2200d7d14a0685a73c6664e52ad47 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:03:14 +0100 Subject: [PATCH 066/109] Added influx checkBucket --- src/helper/influx-checks.js | 16 ++++++++++++++++ src/main.js | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index b44a0fc..f1858f9 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -19,8 +19,24 @@ function checkHealth(influxDb){ }); } +function checkBucket(influxDb, options){ + return new Promise((resolve, reject) => { + new Influx.BucketsAPI(influxDb).getBuckets(options) + .catch((err) => { // Weirdly the influx-Api returns 404 for searches of non-existing buckets + logger.fatal("Could not get bucket:"); + logger.fatal(`Error [${err.code}]:`, err.message); + reject(err); + }).then((res) => { // But an empty list when the bucket exists, but token does not have permission to get details + logger.debug("Bucket found"); + resolve(res); + // Now we know the bucket exists and we have some kind of permission.. but we still dont know if we are able to write to it.. + }); + }); +} + // Specify exports module.exports = { checkHealth, + checkBucket, }; \ No newline at end of file diff --git a/src/main.js b/src/main.js index dec1e82..bb42690 100644 --- a/src/main.js +++ b/src/main.js @@ -26,7 +26,11 @@ if(errorMsg){ const influxDb = new InfluxDB({url: env.INFLUX_URL, token: env.INFLUX_TOKEN}); await InfluxChecks.checkHealth(influxDb) - .catch(exit(1)) - .then((res) => {}); + .then((res) => {return InfluxChecks.checkBucket(influxDb, { + org: env.INFLUX_ORG, + name: env.INFLUX_BUCKET + })}) + .then((res) => {}) + .catch(exit(1)); })(); From c51cfc1b149affb6a8cb64b62b9de91da4e05991 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:35:51 +0100 Subject: [PATCH 067/109] Fixed missing require --- src/main.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.js b/src/main.js index bb42690..47a8a18 100644 --- a/src/main.js +++ b/src/main.js @@ -3,6 +3,7 @@ const logger = require("./helper/logger.js")("main"); const { requireEnvVars } = require("./helper/env.js"); const { exit } = require("process"); const { InfluxDB } = require('@influxdata/influxdb-client'); +const InfluxChecks = require('./helper/influx-checks.js'); /// Setup ENVs const env = process.env; From 3a927688d0d294c2d33b593439da5053e919d433 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:36:35 +0100 Subject: [PATCH 068/109] Changed fatal to error and added error handling to promise catch in main --- src/helper/influx-checks.js | 12 ++++++------ src/main.js | 10 ++++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index f1858f9..857332e 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -8,9 +8,9 @@ function checkHealth(influxDb){ new Influx.HealthAPI(influxDb) // Check influx health .getHealth() .catch((err) => { - logger.fatal("Could not communicate with Influx:"); - logger.fatal(`Error [${err.code}]:`, err.message); - reject(err); + logger.error("Could not communicate with Influx:"); + logger.error(`Error [${err.code}]:`, err.message); + reject(); }) .then((res) => { logger.debug("Server healthy.", "Version: ", res.version); @@ -23,9 +23,9 @@ function checkBucket(influxDb, options){ return new Promise((resolve, reject) => { new Influx.BucketsAPI(influxDb).getBuckets(options) .catch((err) => { // Weirdly the influx-Api returns 404 for searches of non-existing buckets - logger.fatal("Could not get bucket:"); - logger.fatal(`Error [${err.code}]:`, err.message); - reject(err); + logger.error("Could not get bucket:"); + logger.error(`Error [${err.code}]:`, err.message); + reject(); }).then((res) => { // But an empty list when the bucket exists, but token does not have permission to get details logger.debug("Bucket found"); resolve(res); diff --git a/src/main.js b/src/main.js index 47a8a18..0eb1fac 100644 --- a/src/main.js +++ b/src/main.js @@ -31,7 +31,13 @@ if(errorMsg){ org: env.INFLUX_ORG, name: env.INFLUX_BUCKET })}) - .then((res) => {}) - .catch(exit(1)); + .catch((err) => { + if(err) { + logger.error("Error whilst checking influx:"); + logger.error(err); + } + logger.fatal("Setup influx failed!"); + exit(1); + }); })(); From ddf39b94333406e2dbc11e95e0c04f456e0f977d Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:36:50 +0100 Subject: [PATCH 069/109] Fixed require --- src/helper/influx-checks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 857332e..66bdfc5 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -1,4 +1,4 @@ -const logger = require("./helper/logger.js")("influx-checks"); +const logger = require.main.require("./helper/logger.js")("influx-checks"); const Influx = require('@influxdata/influxdb-client-apis'); From 3c29ed20006524cf861d52a16034ad6a1fac37b1 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:37:34 +0100 Subject: [PATCH 070/109] Implemented CheckWriteApi by writing and then deleting a TestPoint --- src/helper/influx-checks.js | 35 +++++++++++++++++++++++++++++++++++ src/main.js | 4 ++++ 2 files changed, 39 insertions(+) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 66bdfc5..34769bb 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -1,5 +1,6 @@ const logger = require.main.require("./helper/logger.js")("influx-checks"); +const { InfluxDB, Point } = require('@influxdata/influxdb-client') const Influx = require('@influxdata/influxdb-client-apis'); @@ -34,9 +35,43 @@ function checkBucket(influxDb, options){ }); } +function checkWriteApi(influxDb, options){ + return new Promise((resolve, reject) => { + const writeApi = influxDb.getWriteApi(options.org, options.bucket); // Get WriteAPI + writeApi.writePoint(new Point("test").tag("ConnectionTest", true).intField("value", 0).timestamp(0)) // Write dummy-point + writeApi.close() + .catch((err) => { + logger.error("Could not get writeApi:"); + logger.error(`Error [${err.code}]:`, err.message); + if(err.code == "not found") logger.fatal("No write-permission?"); + reject(); + }).then((res) => { + logger.debug("Writing ok"); + + // Delete the connection-text-point + new Influx.DeleteAPI(influxDb).postDelete({ + "body": { + "start": 0, + "end": 1, + "predicate": "ConnectionTest==true" + }, + "org": options.org, + "bucket": options.bucket + }) + .catch((err) => { + logger.error("Could not delete ConnectionTest-Point:"); + logger.error(`Error [${err.code}]:`, err.message); + reject(); + }) + .then(resolve()) + }); + }); +} + // Specify exports module.exports = { checkHealth, checkBucket, + checkWriteApi, }; \ No newline at end of file diff --git a/src/main.js b/src/main.js index 0eb1fac..7956c86 100644 --- a/src/main.js +++ b/src/main.js @@ -31,6 +31,10 @@ if(errorMsg){ org: env.INFLUX_ORG, name: env.INFLUX_BUCKET })}) + .then((res) => {return InfluxChecks.checkWriteApi(influxDb, { + org: env.INFLUX_ORG, + name: env.INFLUX_BUCKET + })}) .catch((err) => { if(err) { logger.error("Error whilst checking influx:"); From c5e1bb4c640a1bb5487e23c62f5ac5b5773b9958 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:37:55 +0100 Subject: [PATCH 071/109] Added info-log "ok" --- src/main.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.js b/src/main.js index 7956c86..6a793e5 100644 --- a/src/main.js +++ b/src/main.js @@ -44,4 +44,6 @@ if(errorMsg){ exit(1); }); + logger.info("Influx ok"); + })(); From 37b78e7373d44c410fc137ab0fb7c6b43075f31e Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 18:54:09 +0100 Subject: [PATCH 072/109] Fixed wrong logging-mode used --- src/helper/influx-checks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 34769bb..0d17c30 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -43,7 +43,7 @@ function checkWriteApi(influxDb, options){ .catch((err) => { logger.error("Could not get writeApi:"); logger.error(`Error [${err.code}]:`, err.message); - if(err.code == "not found") logger.fatal("No write-permission?"); + if(err.code == "not found") logger.error("No write-permission?"); reject(); }).then((res) => { logger.debug("Writing ok"); From b0bbf0c71ad08501e9d70235b39ff4ddb482b580 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 19:06:55 +0100 Subject: [PATCH 073/109] Fixed wrong option-subname --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 6a793e5..8fb0a0e 100644 --- a/src/main.js +++ b/src/main.js @@ -33,7 +33,7 @@ if(errorMsg){ })}) .then((res) => {return InfluxChecks.checkWriteApi(influxDb, { org: env.INFLUX_ORG, - name: env.INFLUX_BUCKET + bucket: env.INFLUX_BUCKET })}) .catch((err) => { if(err) { From 2356040572922cfbaa7384a43eb2b7a2ae17f809 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 19:07:27 +0100 Subject: [PATCH 074/109] Move catch into then-onRejected to fix resolving then anyways. --- src/helper/influx-checks.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 0d17c30..9475348 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -40,12 +40,7 @@ function checkWriteApi(influxDb, options){ const writeApi = influxDb.getWriteApi(options.org, options.bucket); // Get WriteAPI writeApi.writePoint(new Point("test").tag("ConnectionTest", true).intField("value", 0).timestamp(0)) // Write dummy-point writeApi.close() - .catch((err) => { - logger.error("Could not get writeApi:"); - logger.error(`Error [${err.code}]:`, err.message); - if(err.code == "not found") logger.error("No write-permission?"); - reject(); - }).then((res) => { + .then((res) => { logger.debug("Writing ok"); // Delete the connection-text-point @@ -64,6 +59,11 @@ function checkWriteApi(influxDb, options){ reject(); }) .then(resolve()) + }, + (err) => { + logger.error("Could not get writeApi:"); + logger.error(`Error [${err.code}]:`, err.message); + reject(); }); }); } From e18de63d7c6ef7117d6abe77fbe9a0a968a5e51c Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 19:14:01 +0100 Subject: [PATCH 075/109] Revert "Move catch into then-onRejected to fix resolving then anyways." This reverts commit 2356040572922cfbaa7384a43eb2b7a2ae17f809. --- src/helper/influx-checks.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 9475348..0d17c30 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -40,7 +40,12 @@ function checkWriteApi(influxDb, options){ const writeApi = influxDb.getWriteApi(options.org, options.bucket); // Get WriteAPI writeApi.writePoint(new Point("test").tag("ConnectionTest", true).intField("value", 0).timestamp(0)) // Write dummy-point writeApi.close() - .then((res) => { + .catch((err) => { + logger.error("Could not get writeApi:"); + logger.error(`Error [${err.code}]:`, err.message); + if(err.code == "not found") logger.error("No write-permission?"); + reject(); + }).then((res) => { logger.debug("Writing ok"); // Delete the connection-text-point @@ -59,11 +64,6 @@ function checkWriteApi(influxDb, options){ reject(); }) .then(resolve()) - }, - (err) => { - logger.error("Could not get writeApi:"); - logger.error(`Error [${err.code}]:`, err.message); - reject(); }); }); } From 271554719e15743f9d6f705b96099cdca7447e5e Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 19:17:24 +0100 Subject: [PATCH 076/109] Remove wrong hint --- src/helper/influx-checks.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 0d17c30..1e6db19 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -43,7 +43,6 @@ function checkWriteApi(influxDb, options){ .catch((err) => { logger.error("Could not get writeApi:"); logger.error(`Error [${err.code}]:`, err.message); - if(err.code == "not found") logger.error("No write-permission?"); reject(); }).then((res) => { logger.debug("Writing ok"); From 873f00b21bea3902da1193bf2517ec682ea0156a Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 19:17:43 +0100 Subject: [PATCH 077/109] Remove deleting point and rename point to something useable --- src/helper/influx-checks.js | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/helper/influx-checks.js b/src/helper/influx-checks.js index 1e6db19..2c77ede 100644 --- a/src/helper/influx-checks.js +++ b/src/helper/influx-checks.js @@ -1,5 +1,6 @@ const logger = require.main.require("./helper/logger.js")("influx-checks"); +const Os = require("os"); const { InfluxDB, Point } = require('@influxdata/influxdb-client') const Influx = require('@influxdata/influxdb-client-apis'); @@ -38,7 +39,7 @@ function checkBucket(influxDb, options){ function checkWriteApi(influxDb, options){ return new Promise((resolve, reject) => { const writeApi = influxDb.getWriteApi(options.org, options.bucket); // Get WriteAPI - writeApi.writePoint(new Point("test").tag("ConnectionTest", true).intField("value", 0).timestamp(0)) // Write dummy-point + writeApi.writePoint(new Point("worker_connectionTest").tag("hostname", Os.hostname())) // Write point writeApi.close() .catch((err) => { logger.error("Could not get writeApi:"); @@ -46,23 +47,7 @@ function checkWriteApi(influxDb, options){ reject(); }).then((res) => { logger.debug("Writing ok"); - - // Delete the connection-text-point - new Influx.DeleteAPI(influxDb).postDelete({ - "body": { - "start": 0, - "end": 1, - "predicate": "ConnectionTest==true" - }, - "org": options.org, - "bucket": options.bucket - }) - .catch((err) => { - logger.error("Could not delete ConnectionTest-Point:"); - logger.error(`Error [${err.code}]:`, err.message); - reject(); - }) - .then(resolve()) + resolve(); }); }); } From 2646c9787e6456f8c2cb845c776eff6b0966fd48 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 21:00:15 +0100 Subject: [PATCH 078/109] Changed key-checking to value-checking This includes e.g. gettings, originally not included in keys --- src/streamHandler/PacketInfluxPointFactory.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/PacketInfluxPointFactory.js b/src/streamHandler/PacketInfluxPointFactory.js index 8d76878..a89415e 100644 --- a/src/streamHandler/PacketInfluxPointFactory.js +++ b/src/streamHandler/PacketInfluxPointFactory.js @@ -37,7 +37,7 @@ class PacketInfluxPointFactory extends Transform{ _transform(packet, encoding, next){ // Create measurements MEASUREMENT_MAP.forEach((objKey, measurement) => { - if(!Object.keys(packet).includes(objKey)) return; + if(packet[objKey] == null) return; let point = new Point(measurement); // Create point From 4ad5eba7e071b9c60928adaf4494f35958265840 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Fri, 26 Nov 2021 21:03:11 +0100 Subject: [PATCH 079/109] Change tag-srtting to recursive by field This will properly set e.g. arrays --- src/streamHandler/PacketInfluxPointFactory.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/streamHandler/PacketInfluxPointFactory.js b/src/streamHandler/PacketInfluxPointFactory.js index a89415e..147658e 100644 --- a/src/streamHandler/PacketInfluxPointFactory.js +++ b/src/streamHandler/PacketInfluxPointFactory.js @@ -42,8 +42,11 @@ class PacketInfluxPointFactory extends Transform{ let point = new Point(measurement); // Create point // Set tags - TAG_LIST.filter(tag => Object.keys(packet).includes(tag)) - .forEach(tag => point.tag(tag, packet[tag])); + TAG_LIST.filter(tag => Object.keys(packet).includes(tag)) // Filter tags available on object + .filter(tag => packet[tag] != null) // Filter tags not falsy on object + .forEach(tag => { + tagObjectRecursively(point, tag, packet[tag]); + }); point.setField('value', packet[objKey]); // Set field @@ -54,6 +57,15 @@ class PacketInfluxPointFactory extends Transform{ } } +function tagObjectRecursively(point, tag, field, suffix = ""){ + if(typeof(field) == "object"){ + // TODO: Convert boolean-arrays like "packet.flags" to key: value + Object.entries(field).map(([key, value]) => { + tagObjectRecursively(point, tag, value, `_${key}${suffix}`); + }); + } + else point.tag(tag+suffix, field); +} /** Mapping for type -> field-method */ const POINT_FIELD_TYPE = new Map([ From a468d7a57b0e6ee247f0fcb6f8336dc384788e37 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:14:50 +0100 Subject: [PATCH 080/109] Start tcpdump process --- src/main.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main.js b/src/main.js index 8fb0a0e..ae6ce01 100644 --- a/src/main.js +++ b/src/main.js @@ -2,6 +2,8 @@ const logger = require("./helper/logger.js")("main"); const { requireEnvVars } = require("./helper/env.js"); const { exit } = require("process"); +const { exec } = require("./helper/exec.js"); + const { InfluxDB } = require('@influxdata/influxdb-client'); const InfluxChecks = require('./helper/influx-checks.js'); @@ -46,4 +48,9 @@ if(errorMsg){ logger.info("Influx ok"); + logger.info("Starting tcpdump.."); + const TCPDUMP_BASECMD = "tcpdump -vvv -e -n -X -s0 -i" + let cmd = `sudo ${TCPDUMP_BASECMD} ${env.WIFI_INTERFACE}`; + + let proc = exec(cmd); })(); From 3c5e941cba05bcd0949a88c10ef239bb47fbdb60 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:15:49 +0100 Subject: [PATCH 081/109] Creating & Attaching streams / Error-logger --- src/main.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main.js b/src/main.js index ae6ce01..5297229 100644 --- a/src/main.js +++ b/src/main.js @@ -7,6 +7,11 @@ const { exec } = require("./helper/exec.js"); const { InfluxDB } = require('@influxdata/influxdb-client'); const InfluxChecks = require('./helper/influx-checks.js'); +const { RegexBlockStream } = require("./streamHandler/RegexBlockStream.js"); +const { PacketStreamFactory } = require("./streamHandler/PacketStreamFactory.js"); +const { PacketInfluxPointFactory } = require("./streamHandler/PacketInfluxPointFactory.js"); +const { InfluxPointWriter } = require("./streamHandler/InfluxPointWriter.js"); + /// Setup ENVs const env = process.env; // Defaults @@ -53,4 +58,17 @@ if(errorMsg){ let cmd = `sudo ${TCPDUMP_BASECMD} ${env.WIFI_INTERFACE}`; let proc = exec(cmd); + logger.debug("Creating & Attaching streams.."); + proc.stdout + .setEncoding("utf8") + .pipe(new RegexBlockStream(/^[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}.*(\n( {4,8}|\t\t?).*){1,}\n/gm)) + .pipe(new PacketStreamFactory()) + .pipe(new PacketInfluxPointFactory()) + .pipe(new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET)); + + logger.debug("Attaching error-logger.."); + proc.stderr.setEncoding("utf8").on("data", (data) => { + logger.error(data); + }); + })(); From 96b52e63a0f35cb2647bfb08d2d5b124cf97b647 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:16:39 +0100 Subject: [PATCH 082/109] Add exit-handler with error-detection --- src/main.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main.js b/src/main.js index 5297229..6660c5e 100644 --- a/src/main.js +++ b/src/main.js @@ -71,4 +71,16 @@ if(errorMsg){ logger.error(data); }); + logger.debug("Attaching exit-handler.."); + proc.on("exit", (code) => { + logger.info(`tcpdump exited code: ${code}`); + if (code) { + logger.fatal(`tcpdump exited with non-zero code: ${code}`); + exit(1); + } + logger.info("Shutdown"); + exit(0); + }); + + logger.info("Startup complete"); })(); From ca3c37be0fe6029d7d06d5572c1cf0780996b52a Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:16:49 +0100 Subject: [PATCH 083/109] Activating strict-mode --- src/main.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.js b/src/main.js index 6660c5e..4212cf4 100644 --- a/src/main.js +++ b/src/main.js @@ -1,3 +1,4 @@ +"use strict"; const logger = require("./helper/logger.js")("main"); const { requireEnvVars } = require("./helper/env.js"); From 54d627d469e446cb5ae2d296b2e308601c91b034 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:20:38 +0100 Subject: [PATCH 084/109] Fix typo --- src/streamHandler/RegexBlockStream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/RegexBlockStream.js b/src/streamHandler/RegexBlockStream.js index ef39a67..30d7646 100644 --- a/src/streamHandler/RegexBlockStream.js +++ b/src/streamHandler/RegexBlockStream.js @@ -13,7 +13,7 @@ class RegexBlockStream extends Transform{ * @param {RegExp} matcher Block-match * @param {boolean} withholdLastBlock When true, the last matches block will not be submitted to prevent submitting incomplete blocks. * @param {boolean} matchAllOnFlush (Only in combination with withholdLastBlock) When enabled, the buffer will be matched on last time on _flush (stream deconstruction) and write any, also incomplete, blocks - * @remarks WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more ressources. + * @remarks WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more resources. */ constructor(matcher, withholdLastBlock = true, matchAllOnFlush = false){ super({ From e39ebaac230a5e3adff4a2b21ef9cb54c3396ffa Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:23:08 +0100 Subject: [PATCH 085/109] Fixed Regex codesmell --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 4212cf4..3726245 100644 --- a/src/main.js +++ b/src/main.js @@ -62,7 +62,7 @@ if(errorMsg){ logger.debug("Creating & Attaching streams.."); proc.stdout .setEncoding("utf8") - .pipe(new RegexBlockStream(/^[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}.*(\n( {4,8}|\t\t?).*){1,}\n/gm)) + .pipe(new RegexBlockStream(/^\d{2}:\d{2}:\d{2}.\d{6}.*(\n( {4,8}|\t\t?).*)+\n/gm)) .pipe(new PacketStreamFactory()) .pipe(new PacketInfluxPointFactory()) .pipe(new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET)); From 54eadf3bc360e7b05909116223ab75c0bdb70975 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:24:18 +0100 Subject: [PATCH 086/109] Fixed Regex codesmells --- src/streamHandler/PacketStreamFactory.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index ea1f5cb..8980f4b 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -65,12 +65,12 @@ class PacketStreamFactory extends Transform{ ?.forEach(match => packet.flags[FLAG_TYPE_MAP[match]] = true) // Set them to true in flags ); - packet.dataRate = Number(data.match(/(?<=^|\s)[0-9]+(\.[0-9]+)?(?=\sMb\/?s($|\s))/i)?.[0]) || null; - packet.frequency = Number(data.match(/(?<=^|\s)[0-9]{4}(?=\sMHz($|\s))/i)?.[0]) || null; + packet.dataRate = Number(data.match(/(?<=^|\s)\d+(\.\d+)?(?=\sMb\/?s($|\s))/i)?.[0]) || null; + packet.frequency = Number(data.match(/(?<=^|\s)\d{4}(?=\sMHz($|\s))/i)?.[0]) || null; - packet.durationMicros = Number(data.match(/(?<=^|\s)[0-9]{1,4}(?=us($|\s))/i)?.[0]) || null; + packet.durationMicros = Number(data.match(/(?<=^|\s)\d{1,4}(?=us($|\s))/i)?.[0]) || null; - packet.signal = Number(data.match(/(?<=^|\s)-[0-9]{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null; + packet.signal = Number(data.match(/(?<=^|\s)-\d{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null; let packetTypeStr = data.match(new RegExp('(?<=^|\\s)('+ PACKET_TYPES_REGEX +')(?=$|\\s)', 'i'))?.[0]; packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: From 8417de5756ab921863e7a7b8e4f3ca166e2e8ac7 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:35:17 +0100 Subject: [PATCH 087/109] Extract checks with if-statements --- src/streamHandler/PacketStreamFactory.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 8980f4b..2a7175d 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -73,9 +73,14 @@ class PacketStreamFactory extends Transform{ packet.signal = Number(data.match(/(?<=^|\s)-\d{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null; let packetTypeStr = data.match(new RegExp('(?<=^|\\s)('+ PACKET_TYPES_REGEX +')(?=$|\\s)', 'i'))?.[0]; - packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: - data.match(/(SA|TA|DA|RA|BSSID):.{17}\s*$/i)? PacketType.NoData: - PacketType.Unknown; + if(packetTypeStr) + packet.packetType = PACKET_TYPE_MAP[packetTypeStr]; + else if(data.match(/(SA|TA|DA|RA|BSSID):.{17}\s*$/i)){ + packet.packetType = PacketType.NoData + } + else { + packet.packetType = PacketType.Unknown; + } packet.srcMac = data.match(/(?<=(^|\s)(SA|TA):).{17}(?=$|\s)/i)?.[0] ?? null; From 227ba127f8218ea93e846d7a98cb629458301c6f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:36:21 +0100 Subject: [PATCH 088/109] Extract packetType handling in own function --- src/streamHandler/PacketStreamFactory.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 2a7175d..6fd6bb8 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -88,6 +88,12 @@ class PacketStreamFactory extends Transform{ packet.bssid = data.match(/(?<=(^|\s)BSSID:).{17}(?=$|\s)/i)?.[0] ?? null; + packet = _handleHeaderPacketType(packet); + + return packet; + } + + _handleHeaderPacketType(packet){ // Cover special cases with more data let newPacket; switch(packet.packetType){ @@ -126,6 +132,12 @@ class PacketStreamFactory extends Transform{ packet.payloadData = hexConv.hexToBytes(data.match(/(?<=\s)([A-F0-9]{1,4}(?=\s))/igm)?.join('') ?? ''); packet.payloadData.splice(packet.payloadData.length-4, 4); // Remove FrameCheck sequence + packet = _handlePayloadPacketType(packet, data); + + return packet; + } + + _handlePayloadPacketType(packet) { // Cover special cases with more data let newPacket; switch(packet.packetType){ From 2ae85ababbfdbdb2cd21e4a0d172dcb1712d6108 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:37:28 +0100 Subject: [PATCH 089/109] Extract keyInfo reading to helper-file --- src/helper/wifiStateAnalyzer.js | 22 ++++++++++++++++++++++ src/streamHandler/PacketStreamFactory.js | 17 +++-------------- 2 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 src/helper/wifiStateAnalyzer.js diff --git a/src/helper/wifiStateAnalyzer.js b/src/helper/wifiStateAnalyzer.js new file mode 100644 index 0000000..33a97ae --- /dev/null +++ b/src/helper/wifiStateAnalyzer.js @@ -0,0 +1,22 @@ + +function keyInfoFromRaw(rawKeyInfo) { + return { + "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111, + "KeyType": keyInfoRaw>>3 & 0b1, + "KeyIndex": keyInfoRaw>>4 & 0b11, + "Install": keyInfoRaw>>6 & 0b1, + "KeyACK": keyInfoRaw>>7 & 0b1, + "KeyMIC": keyInfoRaw>>8 & 0b1, + "Secure": keyInfoRaw>>9 & 0b1, + "Error": keyInfoRaw>>10 & 0b1, + "Request": keyInfoRaw>>11 & 0b1, + "EncryptedKeyData": keyInfoRaw>>12 & 0b1, + "SMKMessage": keyInfoRaw>>13 & 0b1, + }; +} + + +// Specify exports +module.exports = { + keyInfoFromRaw, +}; \ No newline at end of file diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 6fd6bb8..06f59a9 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -3,6 +3,7 @@ const { Transform } = require('stream'); const { DateTime } = require("luxon"); const { PacketType, FlagType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket, HandshakePacket, HandshakeStage } = require.main.require('./dto/Packet.js'); const hexConv = require.main.require("./helper/hexConverter.js"); +const wifiStateAnalyser = require.main.require("./helper/wifiStateAnalyzer.js"); const PACKET_TYPE_MAP = { "Beacon": PacketType.Beacon, @@ -146,20 +147,8 @@ class PacketStreamFactory extends Transform{ // Read key-information const keyInfoRaw = (packet.payloadData[0x5]<<0x8) + packet.payloadData[0x6]; - const keyInfo = { - "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111, - "KeyType": keyInfoRaw>>3 & 0b1, - "KeyIndex": keyInfoRaw>>4 & 0b11, - "Install": keyInfoRaw>>6 & 0b1, - "KeyACK": keyInfoRaw>>7 & 0b1, - "KeyMIC": keyInfoRaw>>8 & 0b1, - "Secure": keyInfoRaw>>9 & 0b1, - "Error": keyInfoRaw>>10 & 0b1, - "Request": keyInfoRaw>>11 & 0b1, - "EncryptedKeyData": keyInfoRaw>>12 & 0b1, - "SMKMessage": keyInfoRaw>>13 & 0b1, - }; - + const keyInfo = wifiStateAnalyser.keyInfoFromRaw(keyInfoRaw); + newPacket.handshakeStage = (!keyInfo.Install && keyInfo.KeyACK && !keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[1] : (!keyInfo.Install && !keyInfo.KeyACK && keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[2] : ( keyInfo.Install && keyInfo.KeyACK && keyInfo.KeyMIC && keyInfo.Secure)? HandshakeStage[3] : From 8ff7211f0feead55b4c4270e6b5431bfc3b9ed69 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:50:14 +0100 Subject: [PATCH 090/109] Revert "Extract packetType handling in own function" This reverts commit 227ba127f8218ea93e846d7a98cb629458301c6f. --- src/streamHandler/PacketStreamFactory.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 06f59a9..3dbbe26 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -89,12 +89,6 @@ class PacketStreamFactory extends Transform{ packet.bssid = data.match(/(?<=(^|\s)BSSID:).{17}(?=$|\s)/i)?.[0] ?? null; - packet = _handleHeaderPacketType(packet); - - return packet; - } - - _handleHeaderPacketType(packet){ // Cover special cases with more data let newPacket; switch(packet.packetType){ @@ -133,12 +127,6 @@ class PacketStreamFactory extends Transform{ packet.payloadData = hexConv.hexToBytes(data.match(/(?<=\s)([A-F0-9]{1,4}(?=\s))/igm)?.join('') ?? ''); packet.payloadData.splice(packet.payloadData.length-4, 4); // Remove FrameCheck sequence - packet = _handlePayloadPacketType(packet, data); - - return packet; - } - - _handlePayloadPacketType(packet) { // Cover special cases with more data let newPacket; switch(packet.packetType){ From 31ab10c3e1043f2d0674ecb96d2d5db240e63827 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:57:05 +0100 Subject: [PATCH 091/109] Fix wrong var-name --- src/helper/wifiStateAnalyzer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helper/wifiStateAnalyzer.js b/src/helper/wifiStateAnalyzer.js index 33a97ae..bb71f60 100644 --- a/src/helper/wifiStateAnalyzer.js +++ b/src/helper/wifiStateAnalyzer.js @@ -1,5 +1,5 @@ -function keyInfoFromRaw(rawKeyInfo) { +function keyInfoFromRaw(keyInfoRaw) { return { "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111, "KeyType": keyInfoRaw>>3 & 0b1, From 91c3aca9e2eb80ca8ca0b87e8b26321b5b62aaa1 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 09:57:33 +0100 Subject: [PATCH 092/109] Add function to get handshakeStage from keyInfo --- src/helper/wifiStateAnalyzer.js | 21 +++++++++++++++++++++ src/streamHandler/PacketStreamFactory.js | 10 +++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/helper/wifiStateAnalyzer.js b/src/helper/wifiStateAnalyzer.js index bb71f60..e2622de 100644 --- a/src/helper/wifiStateAnalyzer.js +++ b/src/helper/wifiStateAnalyzer.js @@ -1,3 +1,4 @@ +const { HandshakeStage } = require.main.require('./dto/Packet.js'); function keyInfoFromRaw(keyInfoRaw) { return { @@ -15,8 +16,28 @@ function keyInfoFromRaw(keyInfoRaw) { }; } +const HANDSHAKE_STAGE_KEYINFO = { + "keys": ["Install", "KeyACK", "KeyMIC", "Secure"], + "0100": HandshakeStage[1], + "0010": HandshakeStage[2], + "1111": HandshakeStage[3], + "0011": HandshakeStage[4], +}; +function handshakeStageFromKeyInfo(keyInfo){ + + // Extract compare-keys + let keyData = ""; + for (const key of HANDSHAKE_STAGE_KEYINFO['keys']) { + keyData += keyInfo[key].toString(); + } + + // Get and return stage + return HANDSHAKE_STAGE_KEYINFO[keyData]; +} + // Specify exports module.exports = { keyInfoFromRaw, + handshakeStageFromKeyInfo, }; \ No newline at end of file diff --git a/src/streamHandler/PacketStreamFactory.js b/src/streamHandler/PacketStreamFactory.js index 3dbbe26..9d7c866 100644 --- a/src/streamHandler/PacketStreamFactory.js +++ b/src/streamHandler/PacketStreamFactory.js @@ -135,13 +135,9 @@ class PacketStreamFactory extends Transform{ // Read key-information const keyInfoRaw = (packet.payloadData[0x5]<<0x8) + packet.payloadData[0x6]; - const keyInfo = wifiStateAnalyser.keyInfoFromRaw(keyInfoRaw); - - newPacket.handshakeStage = (!keyInfo.Install && keyInfo.KeyACK && !keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[1] : - (!keyInfo.Install && !keyInfo.KeyACK && keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[2] : - ( keyInfo.Install && keyInfo.KeyACK && keyInfo.KeyMIC && keyInfo.Secure)? HandshakeStage[3] : - (!keyInfo.Install && !keyInfo.KeyACK && keyInfo.KeyMIC && keyInfo.Secure)? HandshakeStage[4] : - null; + const keyInfo = wifiStateAnalyser.keyInfoFromRaw(keyInfoRaw); // Convert + + newPacket.handshakeStage = wifiStateAnalyser.handshakeStageFromKeyInfo(keyInfo); // Get stage break; } if(newPacket) packet = Object.assign(newPacket, packet); From 2d824543d192c52a9936d52f71f1d5558a402841 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:09:49 +0100 Subject: [PATCH 093/109] fixed cmd --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 98ec383..5acf3bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ RUN npm install # Bundle app source COPY ./src/ . -CMD ["npm", "run"] \ No newline at end of file +CMD ["npm", "run", "start"] \ No newline at end of file From e320d8670beeaaeb1791c396b397a7e14a376020 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:10:05 +0100 Subject: [PATCH 094/109] Add installation of package tcpdump --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5acf3bc..ff467e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,10 @@ WORKDIR /usr/src/app COPY package*.json ./ RUN npm install +RUN apt-get update +RUN apt-get -y install \ + tcpdump + # Bundle app source COPY ./src/ . From d9ee804c3b20761f98227802e5b992db90fe4738 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:10:44 +0100 Subject: [PATCH 095/109] Removed src-references as hats the workdir --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4f3a8dd..309915b 100644 --- a/package.json +++ b/package.json @@ -2,10 +2,10 @@ "name": "rfmon-to-influx", "version": "1.0.0", "description": "Writing (mostly meta-) data received in Wireless-Monitor-Mode into an InfluxDB", - "main": "src/main.js", + "main": "main.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "node src/main.js" + "start": "node main.js" }, "repository": { "type": "git", From a681bbd2d296aae2a08b15a630da147d41f9ad28 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:11:11 +0100 Subject: [PATCH 096/109] Removed sudo as container runs as root anyways --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 3726245..9e6fd77 100644 --- a/src/main.js +++ b/src/main.js @@ -56,7 +56,7 @@ if(errorMsg){ logger.info("Starting tcpdump.."); const TCPDUMP_BASECMD = "tcpdump -vvv -e -n -X -s0 -i" - let cmd = `sudo ${TCPDUMP_BASECMD} ${env.WIFI_INTERFACE}`; + let cmd = `${TCPDUMP_BASECMD} ${env.WIFI_INTERFACE}`; let proc = exec(cmd); logger.debug("Creating & Attaching streams.."); From 45a11753de4d7ba942c1f5e65a815a10afcbd6d7 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:16:23 +0100 Subject: [PATCH 097/109] Add own logger for tcpdump --- src/main.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main.js b/src/main.js index 9e6fd77..2c5f785 100644 --- a/src/main.js +++ b/src/main.js @@ -1,5 +1,6 @@ "use strict"; -const logger = require("./helper/logger.js")("main"); +const logFactory = require("./helper/logger.js"); +const logger = logFactory("main"); const { requireEnvVars } = require("./helper/env.js"); const { exit } = require("process"); @@ -68,16 +69,17 @@ if(errorMsg){ .pipe(new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET)); logger.debug("Attaching error-logger.."); + const loggerTcpdump = logFactory("tcpdump"); proc.stderr.setEncoding("utf8").on("data", (data) => { - logger.error(data); + loggerTcpdump.error(data); }); logger.debug("Attaching exit-handler.."); proc.on("exit", (code) => { - logger.info(`tcpdump exited code: ${code}`); + loggerTcpdump.info(`tcpdump exited code: ${code}`); if (code) { - logger.fatal(`tcpdump exited with non-zero code: ${code}`); - exit(1); + loggerTcpdump.fatal(`tcpdump exited with non-zero code: ${code}`); + exit(1); } logger.info("Shutdown"); exit(0); From c27761322c3348172de863d1a8f9b5406e84596f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:18:29 +0100 Subject: [PATCH 098/109] Catch start-error from tcpdump tcpdump writes to stderr at the beginning --- src/main.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 2c5f785..e7a732e 100644 --- a/src/main.js +++ b/src/main.js @@ -71,7 +71,8 @@ if(errorMsg){ logger.debug("Attaching error-logger.."); const loggerTcpdump = logFactory("tcpdump"); proc.stderr.setEncoding("utf8").on("data", (data) => { - loggerTcpdump.error(data); + if(!data.match(/^(tcpdump: )?listening on /i)) // Catch start-error + loggerTcpdump.error(data); }); logger.debug("Attaching exit-handler.."); From 0709db0ddfa804101bc3c92bee983fdf7de9d0b3 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:44:37 +0100 Subject: [PATCH 099/109] Changed exit-log to debug --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index e7a732e..4841b4c 100644 --- a/src/main.js +++ b/src/main.js @@ -77,7 +77,7 @@ if(errorMsg){ logger.debug("Attaching exit-handler.."); proc.on("exit", (code) => { - loggerTcpdump.info(`tcpdump exited code: ${code}`); + loggerTcpdump.debug(`tcpdump exited code: ${code}`); if (code) { loggerTcpdump.fatal(`tcpdump exited with non-zero code: ${code}`); exit(1); From 5a0118aedd4ca88b9969956f6ccfb95c9b1635a9 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:45:15 +0100 Subject: [PATCH 100/109] Added error-handling for process --- src/main.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main.js b/src/main.js index 4841b4c..7065a7d 100644 --- a/src/main.js +++ b/src/main.js @@ -75,6 +75,10 @@ if(errorMsg){ loggerTcpdump.error(data); }); + proc.on("error", (err) => { + loggerTcpdump.error(err); + }); + logger.debug("Attaching exit-handler.."); proc.on("exit", (code) => { loggerTcpdump.debug(`tcpdump exited code: ${code}`); From 1bf761970f016652424d1aeafd1d2bd42ff21b51 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 13:45:36 +0100 Subject: [PATCH 101/109] Added error-handling for regexBlockStream --- src/main.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 7065a7d..512b8a4 100644 --- a/src/main.js +++ b/src/main.js @@ -61,9 +61,10 @@ if(errorMsg){ let proc = exec(cmd); logger.debug("Creating & Attaching streams.."); + let regexBlockStream = new RegexBlockStream(/^\d{2}:\d{2}:\d{2}.\d{6}.*(\n( {4,8}|\t\t?).*)+\n/gm); proc.stdout .setEncoding("utf8") - .pipe(new RegexBlockStream(/^\d{2}:\d{2}:\d{2}.\d{6}.*(\n( {4,8}|\t\t?).*)+\n/gm)) + .pipe(regexBlockStream) .pipe(new PacketStreamFactory()) .pipe(new PacketInfluxPointFactory()) .pipe(new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET)); @@ -75,6 +76,10 @@ if(errorMsg){ loggerTcpdump.error(data); }); + regexBlockStream.on('error', (err) => { + if(err) loggerTcpdump.error(err); + }) + proc.on("error", (err) => { loggerTcpdump.error(err); }); From 0d84079ce10fcd51cd68159402f56f3540051af9 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 14:20:29 +0100 Subject: [PATCH 102/109] Added packetType to tag-fields --- src/streamHandler/PacketInfluxPointFactory.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/streamHandler/PacketInfluxPointFactory.js b/src/streamHandler/PacketInfluxPointFactory.js index 147658e..e21b7b0 100644 --- a/src/streamHandler/PacketInfluxPointFactory.js +++ b/src/streamHandler/PacketInfluxPointFactory.js @@ -9,6 +9,7 @@ const TAG_LIST = [ "bssid", "frequency", "flags", + "packetType", ]; /** Measurement-name and corresponding field-key */ From 4ddbe3f06f2cb8c204dfabe35c992ed30de3f209 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:09:21 +0100 Subject: [PATCH 103/109] Implemented userHelper methods to detect if streams have data before timeout --- src/helper/userHelper.js | 46 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/helper/userHelper.js diff --git a/src/helper/userHelper.js b/src/helper/userHelper.js new file mode 100644 index 0000000..50a1624 --- /dev/null +++ b/src/helper/userHelper.js @@ -0,0 +1,46 @@ +// This file specifies functions to help a user with e.g. configuration-errors + +function detectStreamData(stream, timeout = 5000){ + return new Promise((resolve, reject) => { + let timeoutHandler; + if(timeout){ + timeoutHandler = setTimeout(() => { + reject('timeout'); + remListeners(); + }, + timeout); + } + + function remListeners(){ + stream.removeListener('error', errorHandler); + stream.removeListener('data', dataHandler); + if(timeoutHandler) clearTimeout(timeoutHandler); + } + + function errorHandler(err) { + remListeners(); + } + function dataHandler(data) { + resolve(data); + remListeners(); + } + + stream.on('error', errorHandler); + stream.on('data', dataHandler); + }); +} + +function detectStreamsData(streams, timeout = 5000){ + let promises = []; + streams.forEach((stream) => { + promises.push(detectStreamData(stream, timeout)); + }) + return promises; +} + + +// Specify exports +module.exports = { + detectStreamData, + detectStreamsData, +}; \ No newline at end of file From c28bbaaadad0c5adff8656f19f81d5c86ef22f9d Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:09:59 +0100 Subject: [PATCH 104/109] Moved Stream-creations outside --- src/main.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main.js b/src/main.js index 512b8a4..12f2f69 100644 --- a/src/main.js +++ b/src/main.js @@ -62,12 +62,15 @@ if(errorMsg){ let proc = exec(cmd); logger.debug("Creating & Attaching streams.."); let regexBlockStream = new RegexBlockStream(/^\d{2}:\d{2}:\d{2}.\d{6}.*(\n( {4,8}|\t\t?).*)+\n/gm); + let packetStreamFactory = new PacketStreamFactory(); + let packetInfluxPointFactory = new PacketInfluxPointFactory(); + let influxPointWriter = new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET); proc.stdout .setEncoding("utf8") .pipe(regexBlockStream) - .pipe(new PacketStreamFactory()) - .pipe(new PacketInfluxPointFactory()) - .pipe(new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET)); + .pipe(packetStreamFactory) + .pipe(packetInfluxPointFactory) + .pipe(influxPointWriter); logger.debug("Attaching error-logger.."); const loggerTcpdump = logFactory("tcpdump"); From 8211f55b89a847832376711209f51fcf07d10ade Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:10:42 +0100 Subject: [PATCH 105/109] Added check for tcpdump-process stdout --- src/main.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main.js b/src/main.js index 12f2f69..bfd719d 100644 --- a/src/main.js +++ b/src/main.js @@ -14,6 +14,9 @@ const { PacketStreamFactory } = require("./streamHandler/PacketStreamFactory.js" const { PacketInfluxPointFactory } = require("./streamHandler/PacketInfluxPointFactory.js"); const { InfluxPointWriter } = require("./streamHandler/InfluxPointWriter.js"); +const userHelper = require("./helper/userHelper.js"); + + /// Setup ENVs const env = process.env; // Defaults @@ -87,6 +90,14 @@ if(errorMsg){ loggerTcpdump.error(err); }); + userHelper.detectStreamData(proc.stdout, 10000) // Expect tcpdump-logs to have data after max. 10s + .then(() => { + loggerTcpdump.debug("Got first data"); + }) + .catch((err) => { + if(err == 'timeout') loggerTcpdump.warn("No data after 10s! Wrong configuration?"); + }); + logger.debug("Attaching exit-handler.."); proc.on("exit", (code) => { loggerTcpdump.debug(`tcpdump exited code: ${code}`); From 16388c73e55011adc1ab32ca8e73f77f468faf71 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:11:14 +0100 Subject: [PATCH 106/109] Added additional check for packetStream --- src/main.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main.js b/src/main.js index bfd719d..2a0a0a6 100644 --- a/src/main.js +++ b/src/main.js @@ -90,9 +90,17 @@ if(errorMsg){ loggerTcpdump.error(err); }); + const loggerPacketStream = logFactory("PacketStreamFactory"); userHelper.detectStreamData(proc.stdout, 10000) // Expect tcpdump-logs to have data after max. 10s .then(() => { loggerTcpdump.debug("Got first data"); + userHelper.detectStreamData(packetStreamFactory, 10000) // Expect then to have packets after further 10s + .then(() => { + loggerPacketStream.debug("Got first packet"); + }) + .catch((err) => { + if(err == 'timeout') loggerPacketStream.warn("No packets"); + }); }) .catch((err) => { if(err == 'timeout') loggerTcpdump.warn("No data after 10s! Wrong configuration?"); From 86d2b8c1cfedac0156cef7eeae050872d1780673 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:34:19 +0100 Subject: [PATCH 107/109] Added event for SIGTERM and SIGINT --- src/main.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main.js b/src/main.js index 2a0a0a6..5710070 100644 --- a/src/main.js +++ b/src/main.js @@ -117,5 +117,14 @@ if(errorMsg){ exit(0); }); + // Handle stop-signals for graceful shutdown + function shutdownReq() { + logger.info("Shutdown request received.."); + logger.debug("Stopping subprocess tcpdump, then exiting myself.."); + proc.kill(); // Kill process (send SIGTERM), then upper event-handler will stop self + } + process.on('SIGTERM', shutdownReq); + process.on('SIGINT', shutdownReq); + logger.info("Startup complete"); })(); From 99a3e13d77944a62278a99049c8fa1aa3f44195b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:35:36 +0100 Subject: [PATCH 108/109] Added further exception to error-event from tcpdump --- src/main.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.js b/src/main.js index 5710070..5527113 100644 --- a/src/main.js +++ b/src/main.js @@ -78,8 +78,10 @@ if(errorMsg){ logger.debug("Attaching error-logger.."); const loggerTcpdump = logFactory("tcpdump"); proc.stderr.setEncoding("utf8").on("data", (data) => { - if(!data.match(/^(tcpdump: )?listening on /i)) // Catch start-error - loggerTcpdump.error(data); + if(!data.match(/^(tcpdump: )?listening on /i) || !data.match(/^\d+ packets captured/i)) { // Catch start-error + loggerTcpdump.debug(data); + } + else loggerTcpdump.error(data); }); regexBlockStream.on('error', (err) => { From d14e469ef4121e92dc8b0a16e902ecacfeb17fcc Mon Sep 17 00:00:00 2001 From: Ruakij Date: Mon, 29 Nov 2021 15:35:53 +0100 Subject: [PATCH 109/109] Removed error-logging from RegexBlockStream FIXME --- src/main.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.js b/src/main.js index 5527113..d471c8f 100644 --- a/src/main.js +++ b/src/main.js @@ -84,9 +84,8 @@ if(errorMsg){ else loggerTcpdump.error(data); }); - regexBlockStream.on('error', (err) => { - if(err) loggerTcpdump.error(err); - }) + // FIXME: This is a hacky workaround to not let errors from subprocess bubble up and terminate our process + regexBlockStream.on('error', (err) => {}); proc.on("error", (err) => { loggerTcpdump.error(err);