From a3d7239895d7055df7a65fddeaffa570fcc3f5ec Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 7 Dec 2021 14:42:37 +0100 Subject: [PATCH 01/17] Change shutdown to accept exitCode and kill-signal --- src/main.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main.js b/src/main.js index e41f35e..18f6fa1 100644 --- a/src/main.js +++ b/src/main.js @@ -117,15 +117,20 @@ if(errorMsg){ loggerTcpdump.debug(`tcpdump exited code: ${code}`); if (code) { loggerTcpdump.fatal(`tcpdump exited with non-zero code: ${code}`); - exit(1); + if(!exitCode) exitCode = 1; // When exitCode is 0, set to 1 } logger.info("Shutdown"); - exit(0); + exit(exitCode); }); // Handle stop-signals for graceful shutdown + var exitCode = 0; function shutdownReq() { logger.info("Shutdown request received.."); + shutdown(); + } + function shutdown(code, signal = "SIGTERM"){ + if(code) exitCode = code; logger.debug("Stopping subprocess tcpdump, then exiting myself.."); proc.kill(); // Kill process (send SIGTERM), then upper event-handler will stop self } From ae389c83109374fe74b01063061334afb89f2888 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 7 Dec 2021 14:43:26 +0100 Subject: [PATCH 02/17] Fix signal not being used --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 18f6fa1..6377ebb 100644 --- a/src/main.js +++ b/src/main.js @@ -132,7 +132,7 @@ if(errorMsg){ function shutdown(code, signal = "SIGTERM"){ if(code) exitCode = code; logger.debug("Stopping subprocess tcpdump, then exiting myself.."); - proc.kill(); // Kill process (send SIGTERM), then upper event-handler will stop self + proc.kill(signal); // Kill process, then upper event-handler will stop self } process.on("SIGTERM", shutdownReq); process.on("SIGINT", shutdownReq); From 7c5c37e9b50869f11a2af35504a4da613a1d3693 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 7 Dec 2021 14:43:49 +0100 Subject: [PATCH 03/17] Implemented check if interface is in wrong mode --- src/main.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main.js b/src/main.js index 6377ebb..c334d1b 100644 --- a/src/main.js +++ b/src/main.js @@ -82,9 +82,22 @@ if(errorMsg){ logger.debug("Attaching error-logger.."); const loggerTcpdump = logFactory("tcpdump"); + let linkTypeId; proc.stderr.setEncoding("utf8").on("data", (data) => { if(!data.match(/^(tcpdump: )?listening on /i) || !data.match(/^\d+ packets captured/i)) { // Catch start-error loggerTcpdump.debug(data); + + if(!linkTypeId && data.match(/^(tcpdump: )?listening on/i)){ // Grab first data containing listen-info if proper header was found + const linkType = data.match(/((?<=link-type ))([a-z].*?) \(.*?\)(?=,)/i)[0]; + const linkTypeData = linkType.match(/(\S*) (.*)/i); + const linkTypeId = linkTypeData[1]; + const linkTypeDetail = linkTypeData[2]; + + if(linkTypeId !== "IEEE802_11_RADIO"){ + logger.error(`Interface not in Monitor-mode! (Expected 'IEEE802_11_RADIO', but got '${linkTypeId}')`); + shutdown(1, "SIGKILL"); + } + } } else loggerTcpdump.error(data); }); From 230c75ffd49e557a97d23f3cb184314ce027ca0f Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 7 Dec 2021 14:51:53 +0100 Subject: [PATCH 04/17] Fix match being wrong way around --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index c334d1b..4e35a9c 100644 --- a/src/main.js +++ b/src/main.js @@ -84,7 +84,7 @@ if(errorMsg){ const loggerTcpdump = logFactory("tcpdump"); let linkTypeId; proc.stderr.setEncoding("utf8").on("data", (data) => { - if(!data.match(/^(tcpdump: )?listening on /i) || !data.match(/^\d+ packets captured/i)) { // Catch start-error + if(data.match(/^(tcpdump: )?listening on /i) || data.match(/^\d+ packets captured/i)) { // Catch start-error loggerTcpdump.debug(data); if(!linkTypeId && data.match(/^(tcpdump: )?listening on/i)){ // Grab first data containing listen-info if proper header was found From a1faee8303a3552f2d05485f90a537297933f3e1 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Tue, 7 Dec 2021 14:53:11 +0100 Subject: [PATCH 05/17] Change linkTypeDetail to be written to the upper-scoped-var --- src/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 4e35a9c..bd9398c 100644 --- a/src/main.js +++ b/src/main.js @@ -90,7 +90,7 @@ if(errorMsg){ if(!linkTypeId && data.match(/^(tcpdump: )?listening on/i)){ // Grab first data containing listen-info if proper header was found const linkType = data.match(/((?<=link-type ))([a-z].*?) \(.*?\)(?=,)/i)[0]; const linkTypeData = linkType.match(/(\S*) (.*)/i); - const linkTypeId = linkTypeData[1]; + linkTypeId = linkTypeData[1]; const linkTypeDetail = linkTypeData[2]; if(linkTypeId !== "IEEE802_11_RADIO"){ From 7753c245d27db98900afc0c16b8af53fffd51392 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 12:52:28 +0100 Subject: [PATCH 06/17] Created first big version of documentation --- README.md | 348 ++++++++++++++++++++++++++++++++++++++++++- docs/img/header0.png | Bin 0 -> 13787 bytes 2 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 docs/img/header0.png diff --git a/README.md b/README.md index fd52817..72f59aa 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,347 @@ -# rfmon-to-influx +rfmon-to-influx +================= -Writing (mostly meta-) data received in Wireless-Monitor-Mode into an InfluxDB. \ No newline at end of file +![](docs/img/header0.png) +*Successful Associations, grouped by AP within 24h* + +
+ +Writing (mostly meta-) data received in Wireless-Monitor-Mode into an InfluxDB. + +
+ +Table of contents +================= + +- [1. Description](#1-description) + - [1.1. What kind of data](#11-what-kind-of-data) + - [1.2. Data-Usage](#12-data-usage) + - [1.3. Tools used](#13-tools-used) +- [2. Usage/Installation](#2-usageinstallation) + - [2.1. Prerequisites](#21-prerequisites) + - [2.2. Running with Docker](#22-running-with-docker) + - [2.3. Environment-Variables](#23-environment-variables) +- [3. Data collected](#3-data-collected) + - [3.1. Data-Types](#31-data-types) + - [3.2. Metric-Overview](#32-metric-overview) + - [3.3. Metric-Details](#33-metric-details) + - [3.4. Tag-Overview](#34-tag-overview) + - [3.5. Tag-Details](#35-tag-details) +- [4. Screenshots](#4-screenshots) +- [5. Potential Issues](#5-potential-issues) + - [5.1. Channel/Frequency](#51-channelfrequency) + - [5.2. Technology](#52-technology) + - [5.3. Data protection](#53-data-protection) + - [5.4. Ethical](#54-ethical) + +
+ +# 1. Description + +This Program listens on an Wifi-Interface in Monitor-Mode (rfmon) and logs most actions made into an influx or influx-like time-database. + +
+ +## 1.1. What kind of data + +**Any** packet sent by a router or station nearby is received and its metadata is collected and categorised. + +The host does **not** have to be part of that network. + +
+ +## 1.2. Data-Usage + +The data can be used to identify problems with the wifi-communication nearby +e.g. +- Wifi-Congestion at certain times of the day +- occurring signal-issues + - e.g. due to broken Microwave-Ovens disrupting communications + - or moving big Objects (e.g. Machines) causing signal-reduction. + +
+ +Aswell as gaining knowledge about installed routers and user interaction with them +e.g. +- in a company environment + - Logging presense and activity of interconnected machines + - Finding other Access-Points not allowed due to potential disruption of Production-Lines + +
+ +Other usages might be threat-detection at Wifi-Level +e.g. +- Deauthentication-Attacks +- Bruteforce-Attempts + +
+ +## 1.3. Tools used + +The program uses `tcpdump` for listening in a subProcess and then extract the metadata when packets arrive. + + +
+ +# 2. Usage/Installation + +## 2.1. Prerequisites + +The Wifi-Interface cannot be used elsewhere at the same time e.g. Network-Manager. +(Packet-capture e.g. tcpdump or Wireshark is ok) + +As of this version, the program does **not** set the interface into monitor mode or changes to channels. + +
+ +### 2.1.1. Interface into Monitor-Mode (rfmon) + +You can change into Monitor-mode beforehand with the packages `net-tools` and `wireless-tools`: +```sh +ifconfig down +iwconfig mode Monitor +ifconfig up +``` + +
+ +### 2.1.2. Set/Change channels + +You can set the channel of the interface (if the interface allows this) with the package `wireless-tools`: +```sh +iw dev set channel +``` + +
+ +## 2.2. Running with Docker + +### 2.2.1. Permissions + +The container must run as **root**, to have permission to listen on the wifi-interface. + +
+ +### 2.2.2. docker run + +Either run with docker directly. + +```sh +docker run + -d + --restart unless-stopped + --network host + -e WIFI_INTERFACE="" + -e INFLUX_URL="http://influxdb:8086/" + -e INFLUX_TOKEN="" + -e INFLUX_ORG="" + -e INFLUX_BUCKET="" + ruakij/rfmon-to-influx:2 +``` + +
+ +### 2.2.3. docker-compose + +Or use the more preferred way with docker-compose. + +`docker-compose.yml` + +```yaml +version: '3' + +services: + rfmon: + container_name: rfmon + image: ruakij/rfmon-to-influx:2 + restart: unless-stopped + network_mode: "host" + environment: + - WIFI_INTERFACE="" + - INFLUX_URL="http://influxdb:8086/" + - INFLUX_TOKEN="" + - INFLUX_ORG="" + - INFLUX_BUCKET="" +``` + +And then pull&start the container: +```sh +docker-compose up -d +``` + +
+ +## 2.3. Environment-Variables + +### 2.3.1. Necessary + +Variable|Description +---|--- +`INFLUX_URL` | Url of influx-server +`INFLUX_TOKEN` | Token with write-access +`INFLUX_ORG` | Organisation and.. +`INFLUX_BUCKET` | Bucket to write into + +
+ +### 2.3.2. Optional + +Variable|Default|Description +---|---|--- +`LOGLEVEL` | INFO | Loglevel +`WIFI_INTERFACE` | wlan0 | Token with write-access +~~`HOSTNAME`~~ | ~~Device's Hostname~~ | ~~Hostname to use as global hostname-tag~~ *(Unused)* + +
+ +# 3. Data collected + +8 Metrics are constructed with 6-10 tags identifying them. + +
+ +## 3.1. Data-Types + +Type|Example|Description +---|---|--- +`String` | Wlan | - +`Number` | 0 | Any normal number, positive and negative +`Boolean` | true | true or false values +`MAC` | 12:34:56:78:9A:BC | Address for L2-networks + +
+ +## 3.2. Metric-Overview +--- +
+ +Name|Type|Description +---|---|--- +rfmon_signal_dbm | `Number` (-95 <> -20) | Signal-Level of every Packet in dBm +rfmon_datarate_bytes | `Number` (1 <> 144) | Data-Rate of every Packet in MBit/s +rfmon_ssid_names | `String` (Length: 0-32) | SSIDs of any Packet containing it +rfmon_authenticationtype_info | `String` | Authentication-Type used by Sender +rfmon_associationsuccess_bools | `Boolean` | Result of an Association +rfmon_disassociationreason_info | `String` | Disconnect-Reason from a ST (not always sent) +rfmon_handshakestage_info | `Number` (1 <> 4) | Stage of a handshake (1 and 3 from ST, 2 and 4 from AP) + +
+ +## 3.3. Metric-Details + +### 3.3.1. rfmon_ssid_names +`String` (Length: 0-32) + +SSIDs from ProbeRequest might be empty (probe for any) or in case of Beacon-Frames could be hidden. + +### 3.3.2. rfmon_authenticationtype_info +`String` {OpenSystem_1, OpenSystem_2, Unknown} + +
+ +## 3.4. Tag-Overview +--- +
+ +Name |Type |Description +---|---|--- +srcmac | `MAC` | Sender's MAC-Address (not present in ClearToSend-Packet) +dstmac | `MAC` | Destination's MAC-Address (not present in RequestToSend-Packet) +bssid | `MAC` | AP's MAC-Address +frequency | `Number` | Frequency the packet was captured on in MHz +packetType | `String` | Type of packet +flags_MoreFragments | `Boolean` | Packet is incomplete +flags_Retry | " | Packet is being retried +flags_PwrMgt | " | Sender will not sleep +flags_MoreData | " | More data in send-buffer to be expected +flags_Protected | " | Packet is protected +flags_Order | " | Packet is strictly ordered + +
+ +## 3.5. Tag-Details + +### 3.5.1. frequency +`Number` (2412 <> 2484) + +The frequency corresponds to following wifi-channels: + +Channel|Frequency +---|--- +1 | 2412 +2 | 2417 +3 | 2422 +4 | 2427 +5 | 2432 +6 | 2437 +7 | 2442 +8 | 2447 +9 | 2452 +10 | 2457 +11 | 2462 +12 | 2467 +13 | 2472 +14 | 2484 + +See [Wikipedia - List of WLAN channels - 2.4GHz](https://en.wikipedia.org/wiki/List_of_WLAN_channels#2.4_GHz_(802.11b/g/n/ax)) for more Information. + +### 3.5.2. packettype +`String` + +Type|Sender|Description +---|---|--- +Beacon | AP | Signal its presence and provide synchronisation for Stations +ProbeRequest | ST | Ask if certain RA/SSID is available +ProbeResponse | AP | Directly respond to Request and Signal own presence +Data | Both | Data-packets +RequestToSend | ST | Ask for transmission-time +ClearToSend | RA | Ack transmission-time +Acknowledgment | Both | Ack Data-Packets +BlockAcknowledgment | Both | Ack alot of Data-Packets at once +NoData | Both | Packet without content, typically used to transmit QoS-States +Authentication | Both | Authentication-process to establish identity and set states +AssociationRequest | ST | Register to AP +AssociationResponse | AP | Respond to registering +Disassociation | ST | Actively unregister e.g. to associate with different AP +Handshake | Both | 4-Way-EAPOL-Handshake to generate encryption-keys between participants +Unknown | - | Unknown packets not identified into above types + +
+ +# 4. Screenshots + +
+ +# 5. Potential Issues + +## 5.1. Channel/Frequency + +The System can only monitor one channel at a time which might not be enough cover, +to combat this, more Interfaces and Systems can be deployed. + +This is not entirely unproblematic, as the system cannot currently prevent packages from being inserted more than once. + +
+ +## 5.2. Technology + +Mismatches between sender and receiver-technologies (e.g. MIMO or HT) can cause packets not being logged at all. +Though this should only be a problem for data-packets. + +
+ +## 5.3. Data protection + +Because the system collects any data, this can be problematic, specially in countries with strong data-protection laws. + +A wifi MAC address is likely to be considered as information of an identifiable natural person, e.g. under GDPR Art.4 (1) and its processing may only be done with prior consent or has to be anonymised. + +
+ +## 5.4. Ethical + +The large-scale collection of data for behavioural or movement analysis, especially without consent of the data subject, is highly controversial. + +Metadata that can be used to track precise activities, such as wifi data, is very powerful and should only be collected and used when necessary. + +If this data falls into the hands of a malicious actor, more precise attacks on the targets could be carried out, such as break-insv, behaviour-based discrimination or more successful phishing. diff --git a/docs/img/header0.png b/docs/img/header0.png new file mode 100644 index 0000000000000000000000000000000000000000..90a4b4e77d1addb12ba455e75962ddc1e78690cb GIT binary patch literal 13787 zcmdUW2{_by-}jt4Maj}CgxoDsA$6>SDJ_x{A!JWN*>}b;lS8XegtE;kB>O(LVMxfH zeczX1Fk=j6j9K2_=$!jL=YG!pKF|HU&-Gr{GZ)u?%>0+%@4I|H-_7$|I-0z@g?B?B z5MHh8SM(u}9iJhPADp?jflo5$!!yBQm)ms{PY8tX3+MNT=repG5Xe!8)|E@Q{SuZ& zeBa)lnNS$_J+rv&w3*wbAFo{Li`#MgM?^zEkDkrFP>ZO$p*xPreWZo%dj8?Lv~c%P z%roKqeaZROH5&PlU-s^ZJ)OsGMq?cwiEx_Qv$&X0vss1JS#Ob|v<$qXh$XeO*d6qp z7Y56K+=*ZG;^Msdy2s%FI6S)EeTwtv>rZ-|!{79hIfpABuKfTG&#r`Vwjl6acn3Hf z-6zl4;>F|O!lTNfDhF7U_XrZB89g{lL?Sn?qG(I9`X%a+i(XqN1alE1V`D2!N(!>+ zt1iJ6W~FX=GmIV5S$Ry%kScSoKbaqPe3A}U?G0>_`Q3Toa#bS*a58 zDmETbpDwvn+~BeS-!S-C1tVZ8X4?`4H4b}>BOAmKetP(}LfFP?cn1o-+BPkqkrpo6 zu10wi1f}vzU>So8ieo-~D>H@?A)S$vJZ;LA_ZpUBWpm?}N$g=bHLyosUXeMWDB?Uk z&y|K2*O;s5iLp=e*Wc(h3oP>*Flc8jxKc+^-PV`eSRIYeouWqHF$y!0WwZ3kk~uh0 zNYb^-H&ZrN2Ell4U$07w=*ct|cb~h2PlOMmT`LyyF7ut1lJQ?y^34^2cfB}xCJw<3 zfvg35;(}cNEEdc)J3WnEZCa46RLR$V*oZN6jV@(|yw?=xjs4-iB1yK z_}bJ|PG1T28MLn06i2Q`KX(;FbX!Y`p<3d6N8Vqaa7D>sOFup6wqnug@vdm4)wy8m z7#^I)TqDX!gh-+{`sNS~Ic2Pq@lK2iD~pkA$38^8`fAP7F5Qh9YPgEib!O*O1a#$b ztR;N2vN3okF<|c1_N`4iuDln69NmQ=n1xFmvg+5VG7ZiB42qV@B8q7BsgVTv}PdjQD3RHI-B$ zT7rEe;73Dl8Ji)7JDySr`S1oEQMyaSnzpGaaeUT ziru#1L^NnuLseHHh(%LAQ|Nc+;7ywnAtr?|t>fv6(~kn7-iynjR#t4{bh<={2#TG# zKOA~GpCa&CTyuop^hJ?Ep@<{ydQECu#szSDdwV;(-TvXx&l4l3AQx{ZLR|}LeTH(8<9f0G1t&x@*3lQa8P#jJxV+mqCGR#Haa zT+mgU?`oeNRhzA;Xi)W=*kLaoP^~WFTGTN<;xj**z1RyS2~lYVo1{W~YHizQn+~)k zj`@(=`soq|?z{3v;4|OMP>6tf5>u;!!4S|ukD47nyO(pg4ZDV6a$6iTvfC_%3(LjeW+5wc_+}NhR zk|#mZNuqbGzB4f~5fAuj>J5p1sXN~W0AqyG>aSd4NT*f^8o53|=*iSoC4Q1QQMb9M ztfUBDP-sZr-Hu6t&*}(X%klsKf9h5^)_z?z=<3_r+)|okggW^)TG~I_b%aWkvW^!NI_FVeHr_qPM4qBaP>|SKiU;hb)@Q z7b~blGnf22lqGzL4jaKb7mIGcBMvW zqVE(vXq-Hu?L%ZIk`_xxnscixyWvsmk9S0`f&j!l5cHivNeWy~!o6HHNQVa@7*OVzQDKRq46&gugj)9L&2%L>* zR#+0&P}*$aQD^zlzwPG}hx)AA!bDz2MRh&Vx(umDEu&&#}32?i4T}-Nm(IcRI4qWd`3hnah`@H~o>SrhN^9Satq*-LCOPJoAqho|p0feM8ebb5 zbFz6QLKd?U;=bsDlObcWpLRl!1*2fxHd zsN9>&E(sG>^~)6~uxm@`xSN@=^`Z}KL`P>w7PN8??>sM4Se<^-?H^AJGLW9y_H<9n zpRFzwST{c;gVJ!MiX zuDk$)`Dm!Y5z!Z`Liwg)INA!{Q^)IKUi9G7lnBQFs(9m=!3~@=b0oPdK(f>5S|KEZdWv$Zve#tl6 zpuTsm);%H^W{M%FSzO}6`!(kB;kiRDWfjQ8=A6R{^Gk|@e>=$$LpgAX1kd$|OrzV0 zex0vJ%3Tyzs3;_<(pR14=AfrXE}}|WsZFkYQEASMDKXRm*F5^kas9GdSy>r1tTL^< z+<#rNP(6-yiAo%~%zgE09c8xFY-MKh3rb$T$NBIm31H!4JJMb2?#gH@@deem4x;IOR zjU<-|TE6|3*4_(al2b3(1%RcrbU*zOJmfmAxcd}{&OKr~Z)&pU+4~AzvC8ByMad@u zcBu1?eHJ^{G7taCN#q{QJ>eHJu{4r%CZ;$x@N4@XitN=7D!>POy6G@ zabqQn-Rwe_drG{QsJ0G1o|1T*a zu6oPyPu_QVb&eGz@-?eEyGNg+a^&DZYN~|`Zf5k$A^;mpulbE+Qq`2Cp!xaPQTH=d z8RZMEB{$J5>&y@!T*xz=o#Pwy9%v>x%+RnIo-?$QtZ2xzzw@~2lCSY>j`(+JRfIC0nW5nuV-uzcFpRP`!Mpn?BDMi-GTylJxe1! z*oi0qek%XlokiYET)q|)Yxr|KbbgRr+nM;-Xci@iwad;0Xw+qtG|%g3lN+PAj?8d_TOS8H?n%Sj59s<(=Sj|*u(Cl3Fe{CxlD zv$0mni(jTtrGQ~CbK9fgH4Wmr;{Kl}Y#PF=7VVFDcz6t+I}KKUQ3WV|dUkdccp8B~ z7(lnsAJDl*Dt!4R_RQfHj!lVWrKclVk9R2#D61h>9V${dVt&};6y~v%i&FO`CJ1P3 zuJ15gC8&85rhob6Gx)h0DtOZ9PN~}^cXut3x-CB}DMr;T{=svRmPqdRIWFFl>))BO z`y(gTcoy~#cV+wIr6ocPdone+m6ferBNL-RZWT`m;5=TgW6_GROJcF2JqST8Bh|$w z858Dfj?+j8)OO;0U zMn?V4eF>-Z|4vcLdMSSoqJHz-&e$?QR*wf#j9lPDMa`x5SFJBy%3Mqv5Vma)J&pA1 zd>Cu)Qv(8Df@^^DWV5EI+-mR8HC{2SF}KMD`?d#phTmVzMMhJTk=B6Sk1pGVV*ebZ z{pOKdRQ#h32QNC`l+`3+65h{5PoY|-&~7)g4YmJ;iOL*WR=3+WXL570 zva=sX3veVw?gJDXWo<8C<=*jOhq0x%`)La2^Do_lY<+&tyG!4h^7WtVSHCf3k<&l9 z({owPhnFX}>^>_Q02f`V)%}eEr`+}bi47x%w`y~Dx61{5{R$TF&{NU^io8=^c=3jx zBB%D2os7-;fZQ!|{2!IL|B0ndWAd_$+ef*?10D=~w9qv7gipqZ#Z$|dVk#DlOBY{Q zM~xzn75~72zsdI?4xKypR1DeQQD}!6PVcH;Q#^F=pe_Qn(sQiJ#Sp|fy6}q}iTOPM zq}hJckKQewxV(BnHw$h9*cA(>!~~Qnk*#jt^d{`*#Kd1TKul}`2(+GaBi8{$W(Bb{ zxBMf$Dxa#VIFWwaZp(dM$5e@==9?7SF*fV)mVVY6XmvC8YC~+>TXIL%zy*Uhd%(Sp z8bxO6jrw^`in!|Rk!WGv@6D;$Lok-R$T5WFrT-vT;p4{1e?@#*xBftUM$h~0UWA5r ziApH{o>=@wa9c^pKgXbE7u?_`SMQnXo0iY2NVj7uy+o%|__SLhteejhADZ8}_3iYg zR?fBgt~8Pk)M(gkYHg(4S;1|b2x@qrYIc8jsI@}MnKNb*HX{4}(_~1P%f(H*x$xoZ zm_1seF*Nbom^#=*oM~pvjht&jfmBgg*?6QJHQbDB*kggJaHg;Fo1Ng`)NWUA zw4fDorj9<4kuifSbGhQMpQGk4`ab0xsYCAX{IjV=ZoV)9y1gx;e*Zq)q^_teDn}G{ zcEkk2db$;5lJz-MwBy;qKVSAgL?8df>h!R9)GN+)^L>jIdg1CGbI;K1a;L7GL=|Uq)YO2zuHi3 ztv^3mc5~oaLPCPVuC0U&2@;n-dC}A(R)zX`VhuYUaM-;;8i5QxLSA zu#y2~v9-980|yQqI&eV76njpt;(LzwI};tRx~Bk>g@H;N)w@LB%s47G79Dwuvp#ZH zr*VryUWUIRw*;uDVH83@5zfP59}HMUO3ObLB!QLR7=Ye`%XPBGc?lzSgy5{cgUViMjra{%8LDixEnUP9je3P+^rchbj%01@d zhpRj}SiB7tt|t$tlRE1h`7&lg~`?hOuW^@WSa5YJXW zY2-wXk9e;DSYaMGdhoyFOf&`mx!G_uZgAs$JT#|P|Bm<+pNRsy=ma>#5RXEflt8?? zk#<@9FmUYhV93~RhXXVP;cnSnWZWV+JGj|lAr12F^j0_sN&om)JNQF?=->0e|Lnq7 z3jbIvhEMwk$gfKvdcGcCV_Uq|bw73+OQpS<$8VtO&cX>^$SYt4v_Y=TXj1<>=C$M!oQ%g*QiiTedK3gnwuo1%=i3Rfc|qbCAJIgHb1&Sci? zehglsD#wvBGN&D)pE-a0W9T>Pw!QB-D-HP4MryXy&{F3ndu3vnOMEDOq>y-wE~~y!zs|f{LX*ug*$`eV4jQ(OGp(C8-6*!u3$V zWoybeZnlAB;rPdguoT|=!(F9$zy4~t4#d%!raPAClT|uVzfj>lkM*~IBiBP0R%?TT#Xvt(| zMg_}f2;_K}8=Ml=Bbs*3we$>AR{X-Xl#?7xy_jX22Ic7Rq8+Eg{tq<25>`+J`~Xf$ zZ|4_3e}3c=4TJd2ZE|rrH+^4~Y9IOZ)P?w#cr>NA^m0rq%()R9PS`0zU6R#nZzzDn z7bhOw;GK@!APZB~iS4wSOE_gW{Dn@@<0lw4%R^e=2?X{t-}2yPXT?uemfY5OQKK~I z_2CnIx9>IWbaFFsW&-&8riMcbn+6E~bZ0;0j(Gf`wH&mI-}mb|Sn}`Hhl8%yn;Wl@ z_iHzp8yE8pse5@gv|BAPfc=>T+EzWLqgDOcXw*0Mbcb`<5`B379~;ZA#c0(%sDr+! z8oH9At~uNv^BMvnobayfFcp-1&U`8MQ<}GLyj1rofyQc401%B^fH7Eph9ccvhR<2+ zQW)|HBzkc+<)v19B>5oe6y0UNwTe&XgXZg?vl&@F`XDWi3Gf*dB>tzk1q-xL+QZoslVZ{%TsH2Jg4C0xTb~W_S`ysQk z9ZUzKkUZ=RuZU`b$a;GA+`f~6d-;UQ)~lsS8LeLe?v}?wAP0juvjFTJd__SA5_cz9 z$6fd*7+B8k9XCy_>A4;}u=9*!DZ|VYb1>JzbkwjpXbSMdQJX!4SYz1D-j7!+7t+W@ z>Jcurq4{kp)Zt>Ju*RtjsZZ#fo&=qW9^K2_J4PrAX}o)6GPBVq9l_Gqx@On0+T<5& zV-&F3wXPU>Kc#~fEVfs6klCnV{Bt(Xkq6tQR1PZD{*<rVwehAe zq+dYQhkbIp|NTmGBcb+*44;H5u+B@&6-#6${jF!Lyd}KF+i<(@r@*z!g#AdP{#mWQ zx*#Yz|DhfU&SS66q&r(XN94?;{#Nu<@a-&~dYUTZ)-2T95E{Y1Xs}YVy0M+AwAzzG zR7?-<9Y3`OgL~nVDrDlAgYl?M>nUelF`qsl8E4FDG%if_#K2PuT9eCZz_$LSEM}Ow ztSBs7X^kFyA!FSsIPvN=ub_nieYx8w7-O;F!JU>P-)W{9L;7Z0hp;*jfuXP5&n2^98> zU4&SJU))RY^?5djj>Tbq9V?UwDHbi-S>bRFN)bP?!@|)xAgOEf(}tdy22bxs>Ki|0 zS#$b4?2PMf3C8`od7ju+sX;$Fnut3V;DV0b}kJz)+#yvkdDw)2d4pXz?Ru1P7O`2!=&)VE|T1K+;W$LL{vXv^ET z%MoGC6?VFtSeLQnn4c^MSAZ4r#P2K=Bn4FKq-u|)FRnTzk0_M>=-{@i?=Gfh#xYpC z0W4hEO%<-VyXco6)Cve`5+OjXgtRc;;rAG1++GD~f(4A%5#c2*D=nM(3H?zAs7w*rBAoZwwAS2^kJvXq50@?$E3%ZRat1v2iG2o{dQ<(QR`K^ zpWpbl=AadNf0qU{S|%H<@NInGr_!u0j^{X6o` zV8F-=R(3Y5e4Z|4AenaR5Qmh8k5gUV6t-i>ry|n$)h3qnZ8WXc3-^LXz0?cNAdY}y zW6!C-YWnC$af$>-eIh^U?cNRQte`1u`4;SNt61frzZH?_?32Iodx7d629tAARu*>R zJ-7@qDfvA2t%YIb>5x}ZQ5I$*8r%aW%JRjpS=@}S2(OX%i<#6}3~c#pyA5o}JNt~{ zo$i@u`!TYc4lBd!6KzGR{l#25gozZS0=4?s5bwGo#+aU7fNxBUY4xVa{cwTcJy$4u6iU`tSg*u!c+IoqqA5rcmXH@qu*c z`3`Y}eo!C&6(*oHs@oCsm|_{6xJxcrKCq7AobZTL0(Tde8L?4v@oa@Gmxm06kixP(A4NTboJd$2V+MW zXE;n+wzX~d^M`MuMb28Fos)5vz4j$q*~1DPF=qQ2J%k%CFx-0aJsm5Z*zyDcquCyL z=HNs4yb4J0azK`=?vo`f^H~?z8 zX8P6wJ%S1NHn`}YPIkIf;qzYpK4%Ioa0zYU(F00~F5mODMDO|y$;1FBXXiq19j%A+ zppe+hJ#g*6R7l)vJoK6dMQv)Ufl8rgiA#K#C}oa32A$_L)gJS4+~w}}s4q|bHMm0$ zNtdo)vU{W>dW6Q zw6QJ)!-kI3R)cXIN%r{5?dvclMOo5XpVdZJ7|yKN!7yN(;ydLtXBLVK+6Ga$;KIT} zNmX`>bIBMO4iQAR+*CiaFI;u9-?NS0Xst;U%nTmBKYm7Lv(6GhjK5>O@`fr?K0W-F ztF5t7ZGt9#B#-@SpwONb@Ws|KN4&NgRmy4)EFo;JO=e{oo-q~+9Db$5#kaNDF(@$w z%V5M>(y5=~HFMw|+KBPl^Jqrx1wJP6wN>C+50ihG(m%z%pefb< zrAmAiOle*-5$cOs2J*w|RPQm_{^ zhM2Vd{Af_(eg{)g4EdIcKO`B*?rzRK;xi~D=~CRIz_o~4j*H(Tw{>ue7cRjLvpQZ9Dt@eo+s=r;+T$o*LpXh$6&UYtcGS-;$tI&V^ zNsug-mT)Fc>-8|p?*1<%LQu_xyg!y~StbyT-l6LpBjGRhxRv;$r#z zNhlu*=yNXuf4lHqf8Xy2$7*Y9z8-$+|3W49tR?mwnezED57e9R20fNj`pK`AGsJb7 zdkP)kWVqIJE&l8&nbT6IPMtk<%DiYctnLJu{CIska&4EnVXNE68|TiaYPUE? zKZqG}dAf`3e^MPtn9-Zkx4-0#GpL(b_-r;yE7nVTMgfGAPuJ$D`KTsDcxP+xtqf}Qea^$WIm4q(D zvkQr#va{=?678(>vL>C&Xd!@0aeKa_(04qc023Y;&~WX+1>-l- zV*zh6b8=3?axK85z~Z~-W-BWcN5_S#ffNN<80@#|ZYz@Du(HC!3yr;q+hfkoO_sO4 zvV}Jd;fbs@#svT}94(barmN z&>oIjD~>%RV@gZzYEvd9SmRhrJ??zw`sRbFUS07lRSzISnXyCTMXnnxr;6vyIdB*COmT|fr7wO)}l#_~&hG~%?RBB|6y zCt-uUddJq~ED(g3&|amaDGN_vy7E$rin5qs3s2DGy$2nYmjYim-%|o_W~VG{gyj02 zI~<#9s}QTYvZ~PgZFfuTW>3~hgDr8S%6FulEF%qoUYuY8_|S#e7k}R#A0pu3;EK%yTtkCV5)V($s)6Gbn2&abYZ(F>I&F%ZjZNtrv!W5%fQ+PF_YGA; zm-yha&-dcV_@thLMC!;LbEI8 ztjU4QmasZF){HBk_s9@AN14N$;lfijIm#T{Jpc%F2l)tiLm}tn)A)U^UOi#VX7r)3 z6YSG?)ZX*=^BiB5*}8uPwDTR9@qV!};eRU=wq^Yy`y%wlepu|AH%5liY(nNi3f?&8 zO@OD3`}~JG9+q!ZQ{*K5X8Bq3BDcC188gkXO!iIAY!@AU(^M}is1KucWIJ`SV_`Bh z=ZJgj3~2I(8UlNuX7-nXmZU_;V;($i8I6jLbrlC8%1(8{GaKp{>DPt^HEMNW#A5+5 zdAWi`f-gi`yjaB2);a zlJ&6iSw%#H!t$dZDymkRQ0nN-0T7lZ`K{zz_y87&|3bc!@4rScbEzcHO+88Q9F-Dr z9flK%r1G1Fza&2+`<3ubZaJ!ryVxQS;$KShEx!*Kw8I^7AL-Bc#j)x$tu$W%hEwuv z?o2BCJ*1)hF3#Spcix!`>!XhB;bAObg4xy*OYn~g>-1(Qe}A@pgr%s8m+D;gWzgch z$Nk>8UAjERn+JS9s12I8cN)S)LDk#wBHLo@hSPcSOmg_)xY+m-s^#WW&=j%DP=^9o zI#~zmk!^LykgTECD2DT7S4RGB5NTv@#-IaxBcNV$cW&J(9jYFIfeEL8^NK>Wkf?B# z)#X5bwR^a!aM;-pw^jGcoKdNS?YZU~J#akBt693FskIe4JGXxn?yLWdx)%xR-Ie~% zWeYOFL&c7$?$?+1D*_DRww?-87_eC$WglTbU~&N8vA2cA^cnD*y1BBlvOXM0ihv~v zam<)c;=ZkIbWBVVk}v?~d8=q>yiIgjKCr$HuzpSmF{ui`08pCeB=Bqw40vwg0f2$1 zd(i%w>1lKK0a-Bq3!6gwV_Ee;4>#>x^nV6B`Xjb}<@{So`kSE`SKpeASf@eTE-LIT zy<&{5qStxk{Pgk^jvBnaG+4Pj*@ueSzt+LJGWBrJv z!!VhUZ)KijFs@)()#?K*xniPzO#YND=&uD`<`#we1kAC>Pmf@9C%4~r=L?{x{)rsrB0?rA=Vey;uny5IMamf_V)H@pC*&dCB!XDOIAvW70$hzgfpkxZlww#=T=Ke z{A%l)t~dR`(R X)2;4P>HbQbzqPLFT*=Y6`^*0U-7_th literal 0 HcmV?d00001 From 6b93a02943beacef832ef15c6322ca10c21ddb3a Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 15:47:26 +0100 Subject: [PATCH 07/17] Implement basic LineProtocolWriter --- .../InfluxDbLineProtocolWriter.js | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/streamHandler/InfluxDbLineProtocolWriter.js diff --git a/src/streamHandler/InfluxDbLineProtocolWriter.js b/src/streamHandler/InfluxDbLineProtocolWriter.js new file mode 100644 index 0000000..a47a843 --- /dev/null +++ b/src/streamHandler/InfluxDbLineProtocolWriter.js @@ -0,0 +1,67 @@ +const logger = require.main.require("./helper/logger.js")("InfluxDbLineProtocolWriter"); +const net = require("net"); + +/** + * Get points and write them into influx + */ +class InfluxDbLineProtocolWriter extends net.Socket{ + /** + * + * @param {string} host Host of line-server + * @param {string} port Port of line-server + * @param {object} options Options for further configuration + */ + constructor(host, port, options = {}) { + super(); + + this._host = host; + this._port = port; + + // options defaults + options.autoConnect ??= true; + options.timeout ??= 5000; + options.autoReconnect ??= true; + this._options = options; + + super.setKeepalive(true, 5000); + + // Register auto-Reconnect if enabled + if(this._options.autoReconnect){ + this.on("connect", () => { + logger.debug("Connection established!"); + + if(this._autoReconnectTimeout) + clearInterval(this._autoReconnectTimeout); + this._autoReconnectTimeout = 0; + }); + + this.on("error", (err) => { + logger.error(err.code, "TCP ERROR"); + if(!this._autoReconnectTimeout) + this._autoReconnectTimeout = setInterval(() => { + this.connect(); + }); + }); + } + + // Autoconnect if requested + if(this._options.autoConnect) this.connect(); + } + + get host(){ return this._host; } + get port(){ return this._port; } + + connect(){ + logger.debug("Connecting.."); + super.connect(this._host, this._port); + } + + write(buffer, errorCb){ + return super.write(buffer, errorCb); + } +} + +// Specify exports +module.exports = { + InfluxDbLineProtocolWriter +}; \ No newline at end of file From 024305db434640e2687ec7e2cbbe78b16313dd4b Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:12:11 +0100 Subject: [PATCH 08/17] Fixed host and port wrong way around --- src/streamHandler/InfluxDbLineProtocolWriter.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/streamHandler/InfluxDbLineProtocolWriter.js b/src/streamHandler/InfluxDbLineProtocolWriter.js index a47a843..1aa4449 100644 --- a/src/streamHandler/InfluxDbLineProtocolWriter.js +++ b/src/streamHandler/InfluxDbLineProtocolWriter.js @@ -53,11 +53,7 @@ class InfluxDbLineProtocolWriter extends net.Socket{ connect(){ logger.debug("Connecting.."); - super.connect(this._host, this._port); - } - - write(buffer, errorCb){ - return super.write(buffer, errorCb); + super.connect(this._port, this._host); } } From 7f5e168fda89a711ee6314df467feb4e4cec8289 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:12:35 +0100 Subject: [PATCH 09/17] Added missing autoReconnectBackoffTime --- src/streamHandler/InfluxDbLineProtocolWriter.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/streamHandler/InfluxDbLineProtocolWriter.js b/src/streamHandler/InfluxDbLineProtocolWriter.js index 1aa4449..f0c9a1f 100644 --- a/src/streamHandler/InfluxDbLineProtocolWriter.js +++ b/src/streamHandler/InfluxDbLineProtocolWriter.js @@ -21,6 +21,7 @@ class InfluxDbLineProtocolWriter extends net.Socket{ options.autoConnect ??= true; options.timeout ??= 5000; options.autoReconnect ??= true; + options.autoReconnectBackoffTime ??= 3000; this._options = options; super.setKeepalive(true, 5000); @@ -40,7 +41,8 @@ class InfluxDbLineProtocolWriter extends net.Socket{ if(!this._autoReconnectTimeout) this._autoReconnectTimeout = setInterval(() => { this.connect(); - }); + }, + this._options.autoReconnectBackoffTime); }); } From 4e2ffec656383d2c5c88df9393467c2b55a395bb Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:12:42 +0100 Subject: [PATCH 10/17] Fix typo --- src/streamHandler/InfluxDbLineProtocolWriter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamHandler/InfluxDbLineProtocolWriter.js b/src/streamHandler/InfluxDbLineProtocolWriter.js index f0c9a1f..8eef2e5 100644 --- a/src/streamHandler/InfluxDbLineProtocolWriter.js +++ b/src/streamHandler/InfluxDbLineProtocolWriter.js @@ -24,7 +24,7 @@ class InfluxDbLineProtocolWriter extends net.Socket{ options.autoReconnectBackoffTime ??= 3000; this._options = options; - super.setKeepalive(true, 5000); + super.setKeepAlive(true, 5000); // Register auto-Reconnect if enabled if(this._options.autoReconnect){ From d1cf1d8f7dc599f653dedbe8eb03b5604d0ea0a3 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:12:59 +0100 Subject: [PATCH 11/17] Add connection-state tracking as getter --- src/streamHandler/InfluxDbLineProtocolWriter.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/streamHandler/InfluxDbLineProtocolWriter.js b/src/streamHandler/InfluxDbLineProtocolWriter.js index 8eef2e5..514c2de 100644 --- a/src/streamHandler/InfluxDbLineProtocolWriter.js +++ b/src/streamHandler/InfluxDbLineProtocolWriter.js @@ -24,12 +24,15 @@ class InfluxDbLineProtocolWriter extends net.Socket{ options.autoReconnectBackoffTime ??= 3000; this._options = options; + this._isConnected = false; + super.setKeepAlive(true, 5000); // Register auto-Reconnect if enabled if(this._options.autoReconnect){ this.on("connect", () => { logger.debug("Connection established!"); + this._isConnected = true; if(this._autoReconnectTimeout) clearInterval(this._autoReconnectTimeout); @@ -38,6 +41,8 @@ class InfluxDbLineProtocolWriter extends net.Socket{ this.on("error", (err) => { logger.error(err.code, "TCP ERROR"); + this._isConnected = false; + if(!this._autoReconnectTimeout) this._autoReconnectTimeout = setInterval(() => { this.connect(); @@ -53,6 +58,8 @@ class InfluxDbLineProtocolWriter extends net.Socket{ get host(){ return this._host; } get port(){ return this._port; } + get isConnected(){ return this._isConnected; } + connect(){ logger.debug("Connecting.."); super.connect(this._port, this._host); From f596a99ee6468ed631e8765f49d8683094284209 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:13:34 +0100 Subject: [PATCH 12/17] Implement converter-stream point -> lineProtocol --- .../InfluxPointToLineProtoStream.js | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/streamHandler/InfluxPointToLineProtoStream.js diff --git a/src/streamHandler/InfluxPointToLineProtoStream.js b/src/streamHandler/InfluxPointToLineProtoStream.js new file mode 100644 index 0000000..a8fc40b --- /dev/null +++ b/src/streamHandler/InfluxPointToLineProtoStream.js @@ -0,0 +1,22 @@ +const logger = require.main.require("./helper/logger.js")("InfluxPointToLineProtoStream"); +const { Transform } = require("stream"); + +/** + * Get points and converts them to Line-protocol + */ +class InfluxPointToLineProtoStream extends Transform{ + constructor(){ + super({ + writableObjectMode: true + }); + } + + _transform(point, encoding, next){ + next(null, point.toLineProtocol() +"\n"); + } +} + +// Specify exports +module.exports = { + InfluxPointToLineProtoStream +}; \ No newline at end of file From 101200131278ed2225231aee32bdd2d8ad97faee Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:14:11 +0100 Subject: [PATCH 13/17] Implement env-var-checks --- src/main.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main.js b/src/main.js index bd9398c..610922e 100644 --- a/src/main.js +++ b/src/main.js @@ -25,12 +25,17 @@ const env = process.env; env.LOGLEVEL ??= "INFO"; env.WIFI_INTERFACE ??= "wlan0"; env.HOSTNAME ??= Os.hostname(); + + env.USE_INFLUXDB_LINEPROTOCOL ??= false; } // Required vars -let errorMsg = requireEnvVars([ - "INFLUX_URL", "INFLUX_TOKEN", - "INFLUX_ORG", "INFLUX_BUCKET" -]); +let errorMsg = requireEnvVars( + env.USE_INFLUXDB_LINEPROTOCOL? [ // When lineprotocol is enabled, we need host and port + "INFLUXDB_LINEPROTOCOL_HOST", "INFLUXDB_LINEPROTOCOL_PORT", + ] : [ // When its disabled, influxdb-data + "INFLUX_URL", "INFLUX_TOKEN", + "INFLUX_ORG", "INFLUX_BUCKET" + ]); if(errorMsg){ logger.fatal(errorMsg); exit(1); From c09c6c29fbe4fd0d75785f4f41755a8326f331db Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:15:10 +0100 Subject: [PATCH 14/17] Move InfluxDB to if-block --- src/main.js | 59 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/main.js b/src/main.js index 610922e..fd2199e 100644 --- a/src/main.js +++ b/src/main.js @@ -14,6 +14,7 @@ const { RegexBlockStream } = require("./streamHandler/RegexBlockStream.js"); const { PacketStreamFactory } = require("./streamHandler/PacketStreamFactory.js"); const { PacketInfluxPointFactory } = require("./streamHandler/PacketInfluxPointFactory.js"); const { InfluxPointWriter } = require("./streamHandler/InfluxPointWriter.js"); +const { InfluxDbLineProtocolWriter } = require("./streamHandler/InfluxDbLineProtocolWriter.js"); const userHelper = require("./helper/userHelper.js"); @@ -42,31 +43,41 @@ if(errorMsg){ } (async function() { - logger.info("Setup Influx.."); - const influxDb = new InfluxDB({url: env.INFLUX_URL, token: env.INFLUX_TOKEN}); - - await InfluxChecks.checkHealth(influxDb) - .then((res) => {return InfluxChecks.checkBucket(influxDb, { - org: env.INFLUX_ORG, - name: env.INFLUX_BUCKET - });}) - .then((res) => {return InfluxChecks.checkWriteApi(influxDb, { - org: env.INFLUX_ORG, - bucket: env.INFLUX_BUCKET - });}) - .catch((err) => { - if(err) { - logger.error("Error whilst checking influx:"); - logger.error(err); - } - logger.fatal("Setup influx failed!"); - exit(1); - }); + let pointWriter; + if(!env.USE_INFLUXDB_LINEPROTOCOL){ + logger.info("Setup Influx.."); + const influxDb = new InfluxDB({url: env.INFLUX_URL, token: env.INFLUX_TOKEN}); + + await InfluxChecks.checkHealth(influxDb) + .then((res) => {return InfluxChecks.checkBucket(influxDb, { + org: env.INFLUX_ORG, + name: env.INFLUX_BUCKET + });}) + .then((res) => {return InfluxChecks.checkWriteApi(influxDb, { + org: env.INFLUX_ORG, + bucket: env.INFLUX_BUCKET + });}) + .catch((err) => { + if(err) { + logger.error("Error whilst checking influx:"); + logger.error(err); + } + logger.fatal("Setup influx failed!"); + exit(1); + }); + + logger.debug("Get WriteApi & set default-hostname to", `'${env.HOSTNAME}'`); + const influxWriteApi = influxDb.getWriteApi(env.INFLUX_ORG, env.INFLUX_BUCKET, "us"); + //influxWriteApi.useDefaultTags({"hostname": env.HOSTNAME}); - logger.debug("Get WriteApi & set default-hostname to", `'${env.HOSTNAME}'`); - const influxWriteApi = influxDb.getWriteApi(env.INFLUX_ORG, env.INFLUX_BUCKET, "us"); - //influxWriteApi.useDefaultTags({"hostname": env.HOSTNAME}); - logger.info("Influx ok"); + pointWriter = new InfluxPointWriter(influxWriteApi); + + logger.info("Influx ok"); + } + else { + logger.info("Setup Influxdb-LineProtocol.."); + + } logger.info("Starting tcpdump.."); const TCPDUMP_BASECMD = "tcpdump -vvv -e -n -X -s0 -i"; From 2f84bb4408008db9d2d40b48b10181d94a754737 Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:15:31 +0100 Subject: [PATCH 15/17] Implement connection-checking --- src/main.js | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/main.js b/src/main.js index fd2199e..3382e03 100644 --- a/src/main.js +++ b/src/main.js @@ -15,6 +15,7 @@ const { PacketStreamFactory } = require("./streamHandler/PacketStreamFactory.js" const { PacketInfluxPointFactory } = require("./streamHandler/PacketInfluxPointFactory.js"); const { InfluxPointWriter } = require("./streamHandler/InfluxPointWriter.js"); const { InfluxDbLineProtocolWriter } = require("./streamHandler/InfluxDbLineProtocolWriter.js"); +const { InfluxPointToLineProtoStream } = require("./streamHandler/InfluxPointToLineProtoStream.js"); const userHelper = require("./helper/userHelper.js"); @@ -77,6 +78,37 @@ if(errorMsg){ else { logger.info("Setup Influxdb-LineProtocol.."); + let lineProtocolWriter = new InfluxDbLineProtocolWriter(env.INFLUXDB_LINEPROTOCOL_HOST, env.INFLUXDB_LINEPROTOCOL_PORT); + + logger.debug("Create PointToLineProto and pipe to LineProtocolWriter"); + pointWriter = new InfluxPointToLineProtoStream(); + pointWriter + .setEncoding("utf8") + .pipe(lineProtocolWriter); + + logger.debug("Waiting for connection.."); + await new Promise((resolve, reject) => { + lineProtocolWriter.once("connect", () => { + resolve(); + }); + lineProtocolWriter.once("error", (err) => { + reject(err); + }); + setTimeout(() => { // After timeout, reject promise + reject("Timeout whilst waiting to connect"); + }, 6500); + }) + .then(() => { + logger.info("Influxdb-LineProtocol ok"); + }) + .catch((err) => { + if(err) { + logger.error("Error whilst checking Influxdb-LineProtocol:"); + logger.error(err); + } + logger.fatal("Setup Influxdb-LineProtocol failed!"); + exit(1); + }); } logger.info("Starting tcpdump.."); @@ -88,13 +120,12 @@ if(errorMsg){ let regexBlockStream = new RegexBlockStream(/^\d{2}:\d{2}:\d{2}.\d{6}.*(\n( {4,8}|\t\t?).*)+\n/gm); let packetStreamFactory = new PacketStreamFactory(); let packetInfluxPointFactory = new PacketInfluxPointFactory(); - let influxPointWriter = new InfluxPointWriter(influxWriteApi); proc.stdout .setEncoding("utf8") .pipe(regexBlockStream) .pipe(packetStreamFactory) .pipe(packetInfluxPointFactory) - .pipe(influxPointWriter); + .pipe(pointWriter); logger.debug("Attaching error-logger.."); const loggerTcpdump = logFactory("tcpdump"); From ae4f2f13f0b8d3d9d3ebaa3aab93209e8fac43ee Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:32:24 +0100 Subject: [PATCH 16/17] Remove screenshot-section, it will be readded later --- README.md | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 72f59aa..e8423cd 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,11 @@ Table of contents - [3.3. Metric-Details](#33-metric-details) - [3.4. Tag-Overview](#34-tag-overview) - [3.5. Tag-Details](#35-tag-details) -- [4. Screenshots](#4-screenshots) -- [5. Potential Issues](#5-potential-issues) - - [5.1. Channel/Frequency](#51-channelfrequency) - - [5.2. Technology](#52-technology) - - [5.3. Data protection](#53-data-protection) - - [5.4. Ethical](#54-ethical) +- [4. Potential Issues](#4-potential-issues) + - [4.1. Channel/Frequency](#41-channelfrequency) + - [4.2. Technology](#42-technology) + - [4.3. Data protection](#43-data-protection) + - [4.4. Ethical](#44-ethical)
@@ -308,13 +307,9 @@ Unknown | - | Unknown packets not identified into above types
-# 4. Screenshots +# 4. Potential Issues -
- -# 5. Potential Issues - -## 5.1. Channel/Frequency +## 4.1. Channel/Frequency The System can only monitor one channel at a time which might not be enough cover, to combat this, more Interfaces and Systems can be deployed. @@ -323,14 +318,14 @@ This is not entirely unproblematic, as the system cannot currently prevent packa
-## 5.2. Technology +## 4.2. Technology Mismatches between sender and receiver-technologies (e.g. MIMO or HT) can cause packets not being logged at all. Though this should only be a problem for data-packets.
-## 5.3. Data protection +## 4.3. Data protection Because the system collects any data, this can be problematic, specially in countries with strong data-protection laws. @@ -338,7 +333,7 @@ A wifi MAC address is likely to be considered as information of an identifiable
-## 5.4. Ethical +## 4.4. Ethical The large-scale collection of data for behavioural or movement analysis, especially without consent of the data subject, is highly controversial. From 69e910428e1d8ab9f3c46a9b6c4579fd8369040e Mon Sep 17 00:00:00 2001 From: Ruakij Date: Thu, 9 Dec 2021 17:57:15 +0100 Subject: [PATCH 17/17] Added documentation for LineProtocol export --- README.md | 88 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e8423cd..33acff7 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,9 @@ Table of contents - [1.3. Tools used](#13-tools-used) - [2. Usage/Installation](#2-usageinstallation) - [2.1. Prerequisites](#21-prerequisites) - - [2.2. Running with Docker](#22-running-with-docker) - - [2.3. Environment-Variables](#23-environment-variables) + - [2.2. Choosing an Export-Method](#22-choosing-an-export-method) + - [2.3. Running with Docker](#23-running-with-docker) + - [2.4. Environment-Variables](#24-environment-variables) - [3. Data collected](#3-data-collected) - [3.1. Data-Types](#31-data-types) - [3.2. Metric-Overview](#32-metric-overview) @@ -113,18 +114,34 @@ iw dev set channel
-## 2.2. Running with Docker +## 2.2. Choosing an Export-Method -### 2.2.1. Permissions +The system allows exporting directly into [InfluxDB](https://docs.influxdata.com/influxdb) version >= 2.0 or into any system using the [InfluxDb-Line-Protocol](https://docs.influxdata.com/influxdb/v2.1/reference/syntax/line-protocol/) e.g. [QuestDB](https://questdb.io/) over TCP. + +As of writing (using InfluxDB v2.1 and using the *flux*-language), the data written by this system was a bit too much for InfluxDB and it struggled very quickly on a fairly beefy machine. + +Thats why the additional LineProtocol-Export-Method was added. Freedom of choice of the Time-Database. + +
+ +If you want to use the InfluxDB-Line-Protocol, simply set the environment variable `USE_INFLUXDB_LINEPROTOCOL` to `true` along with the-other necessary Host and Port-variables. + +
+ +## 2.3. Running with Docker + +### 2.3.1. Permissions The container must run as **root**, to have permission to listen on the wifi-interface.
-### 2.2.2. docker run +### 2.3.2. docker run Either run with docker directly. +
for InfluxDB + ```sh docker run -d @@ -134,18 +151,35 @@ docker run -e INFLUX_URL="http://influxdb:8086/" -e INFLUX_TOKEN="" -e INFLUX_ORG="" - -e INFLUX_BUCKET="" ruakij/rfmon-to-influx:2 ``` +
+ +
for InfluxDB-Line-Protocol + +```sh +docker run + -d + --restart unless-stopped + --network host + -e WIFI_INTERFACE="" + -e USE_INFLUXDB_LINEPROTOCOL="true" + -e INFLUXDB_LINEPROTOCOL_HOST="" + -e INFLUXDB_LINEPROTOCOL_PORT="" + ruakij/rfmon-to-influx:2 +``` +

-### 2.2.3. docker-compose +### 2.3.3. docker-compose Or use the more preferred way with docker-compose. `docker-compose.yml` +
for InfluxDB + ```yaml version: '3' @@ -162,6 +196,28 @@ services: - INFLUX_ORG="" - INFLUX_BUCKET="" ``` +
+ +
for InfluxDB-Line-Protocol + +```yaml +version: '3' + +services: + rfmon: + container_name: rfmon + image: ruakij/rfmon-to-influx:2 + restart: unless-stopped + network_mode: "host" + environment: + - WIFI_INTERFACE="" + - USE_INFLUXDB_LINEPROTOCOL="true" + - INFLUXDB_LINEPROTOCOL_HOST="" + - INFLUXDB_LINEPROTOCOL_PORT="" +``` +
+ +
And then pull&start the container: ```sh @@ -170,9 +226,11 @@ docker-compose up -d
-## 2.3. Environment-Variables +## 2.4. Environment-Variables -### 2.3.1. Necessary +### 2.4.1. Necessary + +
for InfluxDB Variable|Description ---|--- @@ -180,10 +238,20 @@ Variable|Description `INFLUX_TOKEN` | Token with write-access `INFLUX_ORG` | Organisation and.. `INFLUX_BUCKET` | Bucket to write into +
+ +
for InfluxDB-Line-Protocol + +Variable|Description +---|--- +`USE_INFLUXDB_LINEPROTOCOL` | Enable LineProtocol +`INFLUXDB_LINEPROTOCOL_HOST` | Host and.. +`INFLUXDB_LINEPROTOCOL_PORT` | Port of your server +

-### 2.3.2. Optional +### 2.4.2. Optional Variable|Default|Description ---|---|---