Short intro. I have a small Arduino device network based on MySensors with RFM69 radios, a rPi MySensors<->MQTT gateway, and a Home Assistant running on a PC server as controller.
Often my HA sends multiple commands to turn on/off relays on remote devices and although the messages arrive OK and relays actuate, the gateway does not receive the acknowledge because it starts sending a new message. And so the controller does not receive confirmation of state change on all commands sent.
The setup is based on development branch and no custom modifications.

So after going from simple Atmega328 serial gateway via mqtt, esp8266 mqtt gateway, then a Atsamd21 serial gateway via mqtt, and now the rpi gateway, problem still there.

My current solution is a queue implemented in Node-Red. Messages received form controller over mqtt are queued and sent one at a time. If ack is requested but not received in a fixed amount of time (500ms) the message is put back in queue for a total for 5 times. After 5 retries the message is dropped.

This flow work fine so far, all messages are received and acknowledged in time.

[{"id":"c00f16f5.dde598","type":"tab","label":"Flow 1","disabled":false,"info":""},{"id":"f574f085.8c965","type":"simple-queue","z":"c00f16f5.dde598","name":"","firstMessageBypass":true,"bypassInterval":"0","x":410,"y":160,"wires":[["8f09ef2d.2f651"]]},{"id":"47303c08.4463d4","type":"mqtt in","z":"c00f16f5.dde598","name":"","topic":"mysensors-in-queue/#","qos":"2","datatype":"auto","broker":"65b2b296.362e3c","x":200,"y":100,"wires":[["71271e36.f1b4c"]]},{"id":"8f09ef2d.2f651","type":"function","z":"c00f16f5.dde598","name":"process mys","func":"msg.queueLoops++;\nvar m = msg.topic;\nm = m.split('/');\nmsg.retain = false;\nflow.set('node_id',m[1]);\nflow.set('child_id',m[2]);\nflow.set('command',m[3]);\nflow.set('ack',m[4]);\nflow.set('type',m[5]);\nflow.set('payload',msg.payload);\nflow.set('receivedAt',msg.receivedAt);\nif (m[4] == \"0\" ){\n    //message does not requrire acknowladge\n    flow.set('ackRequested',0);\n    //msg.message=\"no ackRequested\";\n    node.log(Date.now() + \":\" + msg.receivedAt +\"/\" + msg.topic + \" \" + msg.payload + \" - noAck\");\n} \nelse\n{\n    //message acknowladge requested\n    flow.set('ackRequested',1);\n    msg.message=\"ackRequested!\";\n    node.log(Date.now() + \":\" + msg.receivedAt +\"/\" + msg.topic + \" \" + msg.payload + \" - ackRequested\");\n}\nmsg.topic = msg.topic.replace('mysensors-in-queue', 'mysensors-in');\nreturn msg;","outputs":1,"noerr":0,"x":570,"y":160,"wires":[["8b425bd3.630458","9ede4d44.22f1f"]]},{"id":"8b425bd3.630458","type":"function","z":"c00f16f5.dde598","name":"check ack","func":"if (flow.get('ackRequested') === 0){\n    //no ack requested, trigger new message from queue. send only trigger command\n    msg = {trigger : 1};\n    return [msg,null];\n}\nelse{\n    //ack requested, start timer and wait for reply\n    msg.mqttPayload = msg.payload\n    return [null,msg];\n}","outputs":2,"noerr":0,"x":760,"y":160,"wires":[["e2913b85.f1b368"],["d843d4a.9199228"]]},{"id":"fd5a5b49.3ec838","type":"mqtt in","z":"c00f16f5.dde598","name":"","topic":"mysensors-out/#","qos":"2","datatype":"auto","broker":"65b2b296.362e3c","x":180,"y":400,"wires":[["c8e82d0.58cabd"]]},{"id":"9ede4d44.22f1f","type":"mqtt out","z":"c00f16f5.dde598","name":"mysensors-in","topic":"","qos":"","retain":"","broker":"65b2b296.362e3c","x":770,"y":100,"wires":[]},{"id":"e2913b85.f1b368","type":"link out","z":"c00f16f5.dde598","name":"Trigger","links":["c2fcc29e.2f944"],"x":1035,"y":160,"wires":[]},{"id":"c2fcc29e.2f944","type":"link in","z":"c00f16f5.dde598","name":"Trigger","links":["e2913b85.f1b368"],"x":255,"y":160,"wires":[["f574f085.8c965"]]},{"id":"d843d4a.9199228","type":"trigger","z":"c00f16f5.dde598","op1":"","op2":"true","op1type":"nul","op2type":"bool","duration":"500","extend":false,"units":"ms","reset":"","bytopic":"all","name":"timeout","x":660,"y":260,"wires":[["956936f9.17acd8"]]},{"id":"956936f9.17acd8","type":"function","z":"c00f16f5.dde598","name":"check status","func":"var maxLoops = 5; \nif (msg.payload === true){\n    //we got a timeout, no ack received\n    if (msg.queueLoops < maxLoops) {\n        //put message back in queue\n        //check queueLopps to avoid a loop and trigger new message from queue\n        msg.payload = msg.mqttPayload;\n        flow.set('ackRequested',0);\n        node.log(msg.topic + \" \" + msg.payload + \" - ack NOK\");\n        return msg;\n    }\n    else{\n        node.log(msg.topic + \" \" + msg.payload + \" - dropped\");\n        msg = {trigger : 1};\n        return msg; \n    }\n}\nif (msg.reset == 1){\n    //ack received\n    //timeout timer canceled\n    //trigger new message from queue\n    msg = {trigger : 1};\n    return msg; \n}\nreturn null;","outputs":1,"noerr":0,"x":850,"y":260,"wires":[["38d0af8e.532d2","e2913b85.f1b368"]]},{"id":"38d0af8e.532d2","type":"function","z":"c00f16f5.dde598","name":"","func":"//simple queue discards a message if trigger propery is set\n//so we send a delayed trigger if \nif (typeof msg.topic  != \"undefined\") {\n    msg = {trigger : 1};\n    return msg;\n}","outputs":1,"noerr":0,"x":790,"y":360,"wires":[["c9ee7809.97a808"]]},{"id":"c9ee7809.97a808","type":"delay","z":"c00f16f5.dde598","name":"","pauseType":"delay","timeout":"1","timeoutUnits":"milliseconds","rate":"1","nbRateUnits":"1","rateUnits":"second","randomFirst":"1","randomLast":"5","randomUnits":"seconds","drop":false,"x":950,"y":360,"wires":[["e2913b85.f1b368"]]},{"id":"71271e36.f1b4c","type":"function","z":"c00f16f5.dde598","name":"Add info","func":"//we need a way to track how many times a message has been throw the queue to avoid an infinite loop\n//set received time of message\nmsg.receivedAt = Date.now();\n//set number of queue count to 0 for new message\nmsg.queueLoops = 0;\nreturn msg;","outputs":1,"noerr":0,"x":420,"y":100,"wires":[["f574f085.8c965"]]},{"id":"c8e82d0.58cabd","type":"function","z":"c00f16f5.dde598","name":"compare","func":"var m = msg.topic;\n\tm = m.split('/');\nif (flow.get('ackRequested') === 1){\n    if ((flow.get('node_id') === m[1]) &&\n        (flow.get('child_id') === m[2]) &&\n        (flow.get('command') === m[3]) &&\n        (flow.get('ack') === m[4]) &&\n        (flow.get('type') === m[5]) &&\n        (flow.get('payload') === msg.payload)){\n            //acknoladge received\n            flow.set('ackRequested',0);\n            msg.reset = 1;\n            node.log(  Date.now() +\":\"+\n                        flow.get('receivedAt',msg.receivedAt) +\"/\"+\n                        msg.topic + \" \" + msg.payload + \" - ack OK\");\n            msg.delay = Date.now() - flow.get('receivedAt',msg.receivedAt) ;\n            msg.node_id =  m[1];\n            return msg;\n        }\n}\nelse\n    msg.error = 1;\nreturn null;\n","outputs":1,"noerr":0,"x":480,"y":400,"wires":[["d843d4a.9199228","956936f9.17acd8"]]},{"id":"65b2b296.362e3c","type":"mqtt-broker","z":"","name":"","broker":"192.168.1.27","port":"1883","clientid":"","usetls":false,"compatmode":false,"keepalive":"60","cleansession":true,"birthTopic":"","birthQos":"0","birthPayload":"","closeTopic":"","closeQos":"0","closePayload":"","willTopic":"","willQos":"0","willPayload":""}]

requires node-red-contrib-simple-message-queue

Hello David,

In plumbing usually we don't have the sensor directly into the water, it is put inside a copper sheath. You should find them at hardware store. Some are short, some are long (for water tanks), and some even have special accessory for cable fixing (second image).
You need to put in a T designed for your pipe type.


You can use thermal paste to increase the thermal conductivity if you want.

I don't know your application but from experience I can say, for heating, if pipes are copper/steel or if you have metal fittings, outside is ok. Mean value over a defined period of time is important.

Hope it helps.
Alex

After more investigations I believe the problem is caused by a mqtt server connection timeout.
By default the gateway tries to send the radio message 5 times. If I lower the number of retries , 2 for example, no more loop. Probably now much more then 25 messages cause a loop.

For testing I added _MQTT_client.loop() in transportSendWrite() (MyTransport.cpp) and, no more loop with default 5 retries.

I also have the gateway near the TV (behind it) with the antenna 30-40 cm below it.
The RSSI value drops for all nodes and this drop can successfully be used to detect tv on . With the magnetic antenna on tv's support, communication stops working even for nodes 4 m away, in the same room.

Bootloader is not the problem, you should see the Ftdi Serial device even without a MCU connected.
Did you connect any power supply to the breadboard? Maybe you reversed the polarity?
It would help if you could post a schematic of the circuit and a photo with the actual circuit.

I do not think it is a security breach, because ESP32 has AES256 encryption for flash data, if enabled.
https://docs.espressif.com/projects/esp-idf/en/latest/esp32/security/flash-encryption.html

@toddles I don't think you can trigger an event directly. HA mysensors integration changes the state of entities only.
Your node sends button state on one single sensor id, value dependent on key pressed? And you only send on depressed event not on release?
If so, why not send also a release event, or count the times one button is pressed before sending?

I also have the gateway near the TV (behind it) with the antenna 30-40 cm below it.
The RSSI value drops for all nodes and this drop can successfully be used to detect tv on . With the magnetic antenna on tv's support, communication stops working even for nodes 4 m away, in the same room.

@Snyfir In your first post you say that only one node is mains powered. Instead of modifying the library, why not provide a UPS type power supply for that node. Something with an old phone battery or 1-2 18650 cells.
But I guess the other two already have something like this, they are not battery only.
That way if the power goes down all your repeaters are up.

In MySensors/hal/architecture/AVR/MyHwAVR.cpp
find:

ISR (WDT_vect)
{
}

// Make it weak to allow local sketch implementation by changing it to

ISR(WDT_vect, __attribute__((weak)))
{
}

Make WDT_vect weak #717

Or you can do it like this Compile problem: multiple definition of `__vector_6'
Go to LowPower.cpp, find this lines

ISR (WDT_vect)
{
	// WDIE & WDIF is cleared in hardware upon entering this ISR
	wdt_disable();
}

and comment it

This last solution worked for me to compile with lowpower lib, but I wanted to use Timer2 on and that is incompatible with Mysensors for now.

Yes, I read it. I started using your approach, gateway sensor ids representing nodes ids, but from the controller point of view this is not straight forward, I think. The 2 problems remain somehow the same. How to get the controllers to understand that gateway sensors are not actually sensors but RSSI value, and repeaters breaks my current algorithm.
Thinking more about this, if it is something the library would benefit, lead me to read about the RF24 and NRF5 radios. I only have RFM69 devices (20 nodes) and it offers RSSI out of the box, same as RFM LoRa version and NRF5. But RF24 lacks this feature. So it somewhat depends on how many are using radios with rssi. Library developers included RSSI functionality, so it has to be of interest.
The repeater problem is partially solved because RF24 networks need repeaters, but don't have RSSI. RFM networks should do without repeaters for most applications. For NRF5, I lack this info. But as I passed the rssi from transport to gateway transport, same can be done with actual sender from the header.

Maybe in version 3.0.0 something like this can be implemented? My understanding is that controllers get link quality data the same way they get all other data from devices, on other platforms. Mabey a I_LINK_QUALITY type can be added especially for this information?

for example my zigbee network:
zigbee2mqtt/device {"linkquality":5,"state":"ON"}

For the past few days I tried to understand how the library works internally and if there is any way to get the gateway received message RSSI to the controller. I'd like to know what the signal strength is at the gateway without asking the node.

By default, RSSI value is not passed from the transport layer to the message layer, so the gateway transport layer cannot use it.
The RSSI value is only sent back to the node via echo message (if requested).
The gateway can request this value via I_SIGNAL_REPORT_REQUEST. But since the value is available directly in the gateway, why not pass it to the controller by default?

I hacked the library (2.4.0) so message layer gets the RSSI value from transport layer (RFM69) and gateway transport layer (MQTT) publishes it as I_SIGNAL_REPORT_RESPONSE, no changes in protocol. But this presents at least 2 problems:

1. I_SIGNAL_REPORT_RESPONSE can be received from the node by request and the value depend on the request.
   Example: controller requests TX power %, node responds and gateway pushes to the controller 2 messages with the same topic but different values and units. There is no way to distinguish which is which.
2. Gateway transport layer knows not if the message was routed via repeater or was received directly from node, and publishes the message as received directly from node, but the RSSI value is for the message from repeater. I don't know how this is handled in the library for I_SIGNAL_REPORT_REQUEST/I_SIGNAL_REPORT_RESPONSE.

Is this something of interest to others? If yes how do you see it implemented, using a dedicated type message, or an existing one? I believe it is useful. Zigbee coordinator provides signal strength for every message. Also Felix Rusu's RFM69 library.

@3nibble Your usb diode should be the one with B2 written on it. You can try shorting it.
Do you have a digital multimeter (DMM)? You can measure the voltage between gnd and both lead of the diode.

@3nibble You shorted the power supply. This will not kill your MCU or usb-serial chip. The 5V rail is the USB power rail and usually if you short it, the USB controller disconnects power or limits current to that USB port until the short is removed or it need a reset.

If one board works and another one doesn't you might have a blown fuse on the backside of the board. Pease post a photo with the backside.
The schematic I found shows a Schottky Diode between VUSB and +5V. It might be open circuit because of your short. If you have a DC 6V-12V power supply you can try powering the board through VIN pin.
Or you can short the usb diode with some conductive tweezers.

Bootloader is not the problem, you should see the Ftdi Serial device even without a MCU connected.
Did you connect any power supply to the breadboard? Maybe you reversed the polarity?
It would help if you could post a schematic of the circuit and a photo with the actual circuit.