Removed error-logging from RegexBlockStream FIXME

tcpdump writes to stderr at the beginning

Dockerfile

@@ -7,7 +7,11 @@ WORKDIR /usr/src/app
 COPY package*.json ./
 RUN npm install
 
+RUN apt-get update
+RUN apt-get -y install \
+    tcpdump
+
 # Bundle app source
 COPY ./src/ .
 
-CMD ["npm", "run"]
+CMD ["npm", "run", "start"]

package.json

@@ -2,10 +2,10 @@
   "name": "rfmon-to-influx",
   "version": "1.0.0",
   "description": "Writing (mostly meta-) data received in Wireless-Monitor-Mode into an InfluxDB",
-  "main": "src/main.js",
+  "main": "main.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "node src/main.js"
+    "start": "node main.js"
   },
   "repository": {
     "type": "git",

src/helper/userHelper.js

@@ -0,0 +1,46 @@
+// This file specifies functions to help a user with e.g. configuration-errors
+
+function detectStreamData(stream, timeout = 5000){
+    return new Promise((resolve, reject) => {
+        let timeoutHandler;
+        if(timeout){
+            timeoutHandler = setTimeout(() => {
+                reject('timeout');
+                remListeners();
+            },
+            timeout);
+        }
+
+        function remListeners(){ 
+            stream.removeListener('error', errorHandler);
+            stream.removeListener('data', dataHandler);
+            if(timeoutHandler) clearTimeout(timeoutHandler);
+        }
+
+        function errorHandler(err) {
+            remListeners();
+        }
+        function dataHandler(data) {
+            resolve(data);
+            remListeners();
+        }
+
+        stream.on('error', errorHandler);
+        stream.on('data', dataHandler);
+    });    
+}
+
+function detectStreamsData(streams, timeout = 5000){
+    let promises = [];
+    streams.forEach((stream) => {
+        promises.push(detectStreamData(stream, timeout));
+    })
+    return promises;
+}
+
+
+// Specify exports
+module.exports = {
+    detectStreamData,
+    detectStreamsData,
+};

src/helper/wifiStateAnalyzer.js

@@ -0,0 +1,43 @@
+const { HandshakeStage } = require.main.require('./dto/Packet.js');
+
+function keyInfoFromRaw(keyInfoRaw) {
+    return {
+        "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111,
+        "KeyType":              keyInfoRaw>>3 & 0b1,
+        "KeyIndex":             keyInfoRaw>>4 & 0b11,
+        "Install":              keyInfoRaw>>6 & 0b1,
+        "KeyACK":               keyInfoRaw>>7 & 0b1,
+        "KeyMIC":               keyInfoRaw>>8 & 0b1,
+        "Secure":               keyInfoRaw>>9 & 0b1,
+        "Error":                keyInfoRaw>>10 & 0b1,
+        "Request":              keyInfoRaw>>11 & 0b1,
+        "EncryptedKeyData":     keyInfoRaw>>12 & 0b1,
+        "SMKMessage":           keyInfoRaw>>13 & 0b1,
+    };
+}
+
+const HANDSHAKE_STAGE_KEYINFO = {
+    "keys": ["Install", "KeyACK", "KeyMIC", "Secure"],
+    "0100": HandshakeStage[1],
+    "0010": HandshakeStage[2],
+    "1111": HandshakeStage[3],
+    "0011": HandshakeStage[4],
+};
+function handshakeStageFromKeyInfo(keyInfo){
+
+    // Extract compare-keys
+    let keyData = "";
+    for (const key of HANDSHAKE_STAGE_KEYINFO['keys']) {
+        keyData += keyInfo[key].toString();
+    }
+    
+    // Get and return stage
+    return  HANDSHAKE_STAGE_KEYINFO[keyData];
+}
+
+
+// Specify exports
+module.exports = {
+    keyInfoFromRaw,
+    handshakeStageFromKeyInfo,
+};

src/main.js

@@ -1,10 +1,22 @@
-const logger = require("./helper/logger.js")("main");
+"use strict";
+const logFactory = require("./helper/logger.js");
+const logger = logFactory("main");
 
 const { requireEnvVars } = require("./helper/env.js");
 const { exit } = require("process");
+const { exec } = require("./helper/exec.js");
+
 const { InfluxDB } = require('@influxdata/influxdb-client');
 const InfluxChecks = require('./helper/influx-checks.js');
 
+const { RegexBlockStream } = require("./streamHandler/RegexBlockStream.js");
+const { PacketStreamFactory } = require("./streamHandler/PacketStreamFactory.js");
+const { PacketInfluxPointFactory } = require("./streamHandler/PacketInfluxPointFactory.js");
+const { InfluxPointWriter } = require("./streamHandler/InfluxPointWriter.js");
+
+const userHelper = require("./helper/userHelper.js");
+
+
 /// Setup ENVs
 const env = process.env;
 // Defaults
@@ -46,4 +58,74 @@ if(errorMsg){
 
   logger.info("Influx ok");
 
+  logger.info("Starting tcpdump..");
+  const TCPDUMP_BASECMD = "tcpdump -vvv -e -n -X -s0 -i"
+  let cmd = `${TCPDUMP_BASECMD} ${env.WIFI_INTERFACE}`;
+
+  let proc = exec(cmd);
+  logger.debug("Creating & Attaching streams..");
+  let regexBlockStream = new RegexBlockStream(/^\d{2}:\d{2}:\d{2}.\d{6}.*(\n( {4,8}|\t\t?).*)+\n/gm);
+  let packetStreamFactory = new PacketStreamFactory();
+  let packetInfluxPointFactory = new PacketInfluxPointFactory();
+  let influxPointWriter = new InfluxPointWriter(influxDb, env.INFLUX_ORG, env.INFLUX_BUCKET);
+  proc.stdout
+    .setEncoding("utf8")
+    .pipe(regexBlockStream)
+    .pipe(packetStreamFactory)
+    .pipe(packetInfluxPointFactory)
+    .pipe(influxPointWriter);
+
+  logger.debug("Attaching error-logger..");
+  const loggerTcpdump = logFactory("tcpdump");
+  proc.stderr.setEncoding("utf8").on("data", (data) => {
+    if(!data.match(/^(tcpdump: )?listening on /i) || !data.match(/^\d+ packets captured/i)) {  // Catch start-error
+        loggerTcpdump.debug(data);
+    }
+    else loggerTcpdump.error(data);
+  });
+
+  // FIXME: This is a hacky workaround to not let errors from subprocess bubble up and terminate our process
+  regexBlockStream.on('error', (err) => {});
+
+  proc.on("error", (err) => {
+    loggerTcpdump.error(err);
+  });
+
+  const loggerPacketStream = logFactory("PacketStreamFactory");
+  userHelper.detectStreamData(proc.stdout, 10000)       // Expect tcpdump-logs to have data after max. 10s
+    .then(() => {
+        loggerTcpdump.debug("Got first data");
+        userHelper.detectStreamData(packetStreamFactory, 10000)      // Expect then to have packets after further 10s
+            .then(() => {
+                loggerPacketStream.debug("Got first packet");
+            })
+            .catch((err) => {
+                if(err == 'timeout') loggerPacketStream.warn("No packets");
+            });
+    })
+    .catch((err) => {
+        if(err == 'timeout') loggerTcpdump.warn("No data after 10s! Wrong configuration?");
+    });
+
+  logger.debug("Attaching exit-handler..");
+  proc.on("exit", (code) => {
+    loggerTcpdump.debug(`tcpdump exited code: ${code}`);
+    if (code) {
+        loggerTcpdump.fatal(`tcpdump exited with non-zero code: ${code}`);
+        exit(1);
+    }
+    logger.info("Shutdown");
+    exit(0);
+  });
+
+  // Handle stop-signals for graceful shutdown
+  function shutdownReq() {
+    logger.info("Shutdown request received..");
+    logger.debug("Stopping subprocess tcpdump, then exiting myself..");
+    proc.kill();    // Kill process (send SIGTERM), then upper event-handler will stop self
+  }
+  process.on('SIGTERM', shutdownReq);
+  process.on('SIGINT', shutdownReq);
+
+  logger.info("Startup complete");
 })();

src/streamHandler/PacketInfluxPointFactory.js

@@ -9,6 +9,7 @@ const TAG_LIST = [
     "bssid",
     "frequency",
     "flags",
+    "packetType",
 ];
 
 /** Measurement-name and corresponding field-key */
@@ -37,13 +38,16 @@ class PacketInfluxPointFactory extends Transform{
     _transform(packet, encoding, next){
         // Create measurements
         MEASUREMENT_MAP.forEach((objKey, measurement) => {
-            if(!Object.keys(packet).includes(objKey)) return;
+            if(packet[objKey] == null) return;
 
             let point = new Point(measurement);     // Create point
             
             // Set tags
-            TAG_LIST.filter(tag => Object.keys(packet).includes(tag))
-                    .forEach(tag => point.tag(tag, packet[tag]));
+            TAG_LIST.filter(tag => Object.keys(packet).includes(tag))   // Filter tags available on object
+                    .filter(tag => packet[tag] != null)                         // Filter tags not falsy on object
+                    .forEach(tag => {
+                        tagObjectRecursively(point, tag, packet[tag]);
+                    });
 
             point.setField('value', packet[objKey]);        // Set field
 
@@ -54,6 +58,15 @@ class PacketInfluxPointFactory extends Transform{
     }
 }
 
+function tagObjectRecursively(point, tag, field, suffix = ""){
+    if(typeof(field) == "object"){
+        // TODO: Convert boolean-arrays like "packet.flags" to key: value
+        Object.entries(field).map(([key, value]) => {
+            tagObjectRecursively(point, tag, value, `_${key}${suffix}`);
+        });
+    }
+    else point.tag(tag+suffix, field);
+}
 
 /** Mapping for type -> field-method */
 const POINT_FIELD_TYPE = new Map([

src/streamHandler/PacketStreamFactory.js

@@ -3,6 +3,7 @@ const { Transform } = require('stream');
 const { DateTime } = require("luxon");
 const { PacketType, FlagType, Packet, PacketWithSSID, BeaconPacket, ProbeRequestPacket, ProbeResponsePacket, AuthenticationPacket, AuthenticationType, AssociationResponsePacket, DisassociationPacket, HandshakePacket, HandshakeStage } = require.main.require('./dto/Packet.js');
 const hexConv = require.main.require("./helper/hexConverter.js");
+const wifiStateAnalyser = require.main.require("./helper/wifiStateAnalyzer.js");
 
 const PACKET_TYPE_MAP = {
     "Beacon":           PacketType.Beacon,
@@ -65,17 +66,22 @@ class PacketStreamFactory extends Transform{
             ?.forEach(match => packet.flags[FLAG_TYPE_MAP[match]] = true)       // Set them to true in flags
         );
 
-        packet.dataRate = Number(data.match(/(?<=^|\s)[0-9]+(\.[0-9]+)?(?=\sMb\/?s($|\s))/i)?.[0]) || null;
-        packet.frequency = Number(data.match(/(?<=^|\s)[0-9]{4}(?=\sMHz($|\s))/i)?.[0]) || null;
+        packet.dataRate = Number(data.match(/(?<=^|\s)\d+(\.\d+)?(?=\sMb\/?s($|\s))/i)?.[0]) || null;
+        packet.frequency = Number(data.match(/(?<=^|\s)\d{4}(?=\sMHz($|\s))/i)?.[0]) || null;
 
-        packet.durationMicros = Number(data.match(/(?<=^|\s)[0-9]{1,4}(?=us($|\s))/i)?.[0]) || null;
+        packet.durationMicros = Number(data.match(/(?<=^|\s)\d{1,4}(?=us($|\s))/i)?.[0]) || null;
 
-        packet.signal = Number(data.match(/(?<=^|\s)-[0-9]{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null;
+        packet.signal = Number(data.match(/(?<=^|\s)-\d{2,3}(?=dBm\sSignal($|\s))/i)?.[0]) || null;
 
         let packetTypeStr = data.match(new RegExp('(?<=^|\\s)('+ PACKET_TYPES_REGEX +')(?=$|\\s)', 'i'))?.[0];
-        packet.packetType = packetTypeStr? PACKET_TYPE_MAP[packetTypeStr]: 
-                            data.match(/(SA|TA|DA|RA|BSSID):.{17}\s*$/i)? PacketType.NoData:
-                            PacketType.Unknown;
+        if(packetTypeStr)
+            packet.packetType = PACKET_TYPE_MAP[packetTypeStr];
+        else if(data.match(/(SA|TA|DA|RA|BSSID):.{17}\s*$/i)){
+            packet.packetType = PacketType.NoData
+        }
+        else {
+            packet.packetType = PacketType.Unknown;
+        }
 
         packet.srcMac = data.match(/(?<=(^|\s)(SA|TA):).{17}(?=$|\s)/i)?.[0] ?? null;
 
@@ -129,25 +135,9 @@ class PacketStreamFactory extends Transform{
 
                 // Read key-information
                 const keyInfoRaw = (packet.payloadData[0x5]<<0x8) + packet.payloadData[0x6];
-                const keyInfo = {
-                    "KeyDescriptorVersion": keyInfoRaw>>0 & 0b111,
-                    "KeyType":  keyInfoRaw>>3 & 0b1,
-                    "KeyIndex": keyInfoRaw>>4 & 0b11,
-                    "Install":  keyInfoRaw>>6 & 0b1,
-                    "KeyACK":   keyInfoRaw>>7 & 0b1,
-                    "KeyMIC":   keyInfoRaw>>8 & 0b1,
-                    "Secure":   keyInfoRaw>>9 & 0b1,
-                    "Error":    keyInfoRaw>>10 & 0b1,
-                    "Request":  keyInfoRaw>>11 & 0b1,
-                    "EncryptedKeyData": keyInfoRaw>>12 & 0b1,
-                    "SMKMessage":       keyInfoRaw>>13 & 0b1,
-                };
+                const keyInfo = wifiStateAnalyser.keyInfoFromRaw(keyInfoRaw);   // Convert
 
-                newPacket.handshakeStage =  (!keyInfo.Install &&  keyInfo.KeyACK && !keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[1] :
-                                            (!keyInfo.Install && !keyInfo.KeyACK &&  keyInfo.KeyMIC && !keyInfo.Secure)? HandshakeStage[2] :
-                                            ( keyInfo.Install &&  keyInfo.KeyACK &&  keyInfo.KeyMIC &&  keyInfo.Secure)? HandshakeStage[3] :
-                                            (!keyInfo.Install && !keyInfo.KeyACK &&  keyInfo.KeyMIC &&  keyInfo.Secure)? HandshakeStage[4] :
-                                            null;
+                newPacket.handshakeStage =  wifiStateAnalyser.handshakeStageFromKeyInfo(keyInfo);   // Get stage
                 break;
         }
         if(newPacket) packet = Object.assign(newPacket, packet);

src/streamHandler/RegexBlockStream.js

@@ -13,7 +13,7 @@ class RegexBlockStream extends Transform{
      * @param {RegExp} matcher Block-match
      * @param {boolean} withholdLastBlock When true, the last matches block will not be submitted to prevent submitting incomplete blocks.
      * @param {boolean} matchAllOnFlush (Only in combination with withholdLastBlock) When enabled, the buffer will be matched on last time on _flush (stream deconstruction) and write any, also incomplete, blocks
-     * @remarks WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more ressources.
+     * @remarks WARNING: It should match a clean-block (including e.g. newline)! Otherwise buffer will get dirty and use more and more resources.
      */
     constructor(matcher, withholdLastBlock = true, matchAllOnFlush = false){
         super({