💬 Building a MQTT Gateway

Guillermo Schimmel

@mfalkvidd thanks. I just did that on a dedicated post. It's driving me crazy.

https://forum.mysensors.org/topic/10109/ethernet-mqtt-gateway-slow-to-connect

mfalkvidd

@guillermo-schimmel ok, lets discuss there then so we don't split the effort and information in separate threads.

evb

Hi, receiving an upvote on a relatively old topic of mine, did remember me that this information could help others searching to build a MQTT gateway with a RFM69 radio, a W5100 Ethernet shield and a simple Arduino UNO R3.

https://forum.mysensors.org/topic/10358/solved-mqtt-gateway-based-on-arduino-uno-uno-compatible-w5100-ethernet-shield-rfm69hw-radio

People starting with MySensors, normally read as first step, the documentation available.
Every open source project is structurally a bit different and sometimes it is difficult to know how to contribute to the project.
For example, the open source project HomeAssistant (a possible controller for MySensors) uses a standard git repository for its documentation where anyone can make a pull request with their contribution. The moderators then review each contribution and either accept it or not. You have the official documentation and the 'user guides' documentation.
@mfalkvidd, maybe a workflow to consider? It can help to keep the documentation up to date faster and assess contributions better and faster.
Most open source projects run on volunteers who spend their free time on these projects, so I'm happy to return some knowledge.

In the meantime, I've learned here and realized that under every documentation page, there is a forum topic.
Unfortunately, documentation is usually not really up to date and the topic can be very long. Very hard to read for beginners. :-(

That's why I created some forum topics with my first steps in the Mysensors world to overcome some outdated documentation. But of course these are hard to find among all the other topics and if you don't know anything about their existence, it's even harder.
And like everything, my topic is also already outdated, because, if I am correct, today a W5200 Ethernet shield is existing...

(for your information, my mqtt gateway is still alive and kicking ;-) )

tssk

Does anyone know if it would be possible to add static information that would be available over HTTP (default port 80)?

My idea is to put some kind of description/useful information in the MQTT GW itself. So in case Controller fails. There is some information resource what is the purpose, functionality and how to use it in some limited/emergency way eg. from phone or any computer.

EDIT: Thinking of it - it might also provide some kind of configuration interface...

frits

@tssk Nice idea! This should work, at the cost of some other feature which doesn't fit into ATMega's flash anymore.
Here's my gateway:

#define BUILD_MQTT_GATEWAY // either build mqtt gateway or ethernet gateway
// #define USE_SIGNING
#define MY_RADIO_RF24
// #define MY_RADIO_RFM69
#define WEBFRONTEND // provide a webfrontend at port 80 that shows gateway config
#define MY_DEBUG

#define SN "MySensors W5100 Gateway"
#define SV "0.2"

#define MY_MQTT_PUBLISH_TOPIC_PREFIX "mysgw1-out"
#define MY_MQTT_SUBSCRIBE_TOPIC_PREFIX "mysgw1-in"
#define MY_MQTT_CLIENT_ID "mysgw1"
#define MY_MQTT_CONTROLLER_IP_ADDRESS 192, 168, 99, 20
#define MY_MQTT_PORT 1883
#define MY_MYSGW_PORT 5003

#define MY_IP_ADDRESS 192,168,99,13
#define MY_MAC_ADDRESS 0xDE, 0xAD, 0xBA, 0xBE, 0x99, 0x13
#define MY_SIGNING_ATSHA204_PIN 17
#define MY_RFM69_NETWORKID (200)
#define MY_RFM69_TX_POWER_DBM 20
#define MY_SERIALDEVICE Serial

#if !defined(MY_W5100_SPI_EN) && !defined(ARDUINO_ARCH_SAMD)
  #define MY_SOFTSPI
  #define MY_SOFT_SPI_SCK_PIN (14)
  #define MY_SOFT_SPI_MISO_PIN (16)
  #define MY_SOFT_SPI_MOSI_PIN (15)
#endif

#ifdef MY_RADIO_RF24
  #define MY_RF24_CE_PIN (5)
  #define MY_RF24_CS_PIN (6)
#elif defined MY_RADIO_RFM69
  #define MY_IS_RFM69HW
  #define MY_RFM69_NEW_DRIVER
  #define MY_RF69_SPI_CS (14)
#else
  #error "only rfm24 or rf69 supported"
#endif

#ifdef BUILD_MQTT_GATEWAY
  #define MY_GATEWAY_MQTT_CLIENT
  #define MY_PORT MY_MQTT_PORT
  #define MY_CONTROLLER_IP_ADDRESS MY_MQTT_CONTROLLER_IP_ADDRESS
#else
  #define MY_GATEWAY_W5100
  #define MY_PORT MY_MYSGW_PORT
#endif

#ifdef USE_SIGNING
  #define MY_SIGNING_ATSHA204_PIN 17
  #define MY_SIGNING_ATSHA204 //!< Hardware signing using ATSHA204A
  #define MY_SIGNING_REQUEST_SIGNATURES
#endif

void printIpAddress ( Stream &s, uint8_t a, uint8_t b, uint8_t c, uint8_t d )
{
  s.print ( a );
  s.print ( "." );
  s.print ( b );
  s.print ( "." );
  s.print ( c );
  s.print ( "." );
  s.print ( d );
}

void printMacAddress ( Stream &s, uint8_t a, uint8_t b, uint8_t c, uint8_t d, uint8_t e, uint8_t f )
{
  s.print ( a, HEX );
  s.print ( ":" );
  s.print ( b, HEX );
  s.print ( ":" );
  s.print ( c, HEX );
  s.print ( ":" );
  s.print ( d, HEX );
  s.print ( ":" );
  s.print ( e, HEX );
  s.print ( ":" );
  s.print ( f, HEX );
}

void printConfig ( Stream &s, char * lineBreak )
{
  s.print ( F ( "***********************************************" ) );
  s.print ( lineBreak );
  s.print ( SN );
  s.print ( lineBreak );
  s.print ( "Version " );
  s.print ( SV );
  s.print ( lineBreak );
  s.print ( F("own mac address: ") );
  printMacAddress ( s, MY_MAC_ADDRESS );
  s.print ( lineBreak );
  s.print ( F("own network address: ") );
  printIpAddress ( s, MY_IP_ADDRESS );
  s.print ( lineBreak );
#ifdef BUILD_MQTT_GATEWAY
    s.print ( F("Gateway type: mqtt" ) );
    s.print ( lineBreak );
    s.print ( F ( "mqtt controller address: " ) );
    printIpAddress ( s, MY_CONTROLLER_IP_ADDRESS );
    s.print ( lineBreak );
    s.print ( F ( "mqtt controller port: " ) );
    s.print ( MY_MQTT_PORT );
    s.print ( lineBreak );
    s.print ( F ( "mqtt client id: " ) );
    s.print ( MY_MQTT_CLIENT_ID );
    s.print ( lineBreak );
    s.print ( F ( "mqtt publish prefix: " ) );
    s.print ( MY_MQTT_PUBLISH_TOPIC_PREFIX );
    s.print ( lineBreak );
    s.print ( F ( "mqtt subscribe prefix: " ) );
    s.print ( MY_MQTT_SUBSCRIBE_TOPIC_PREFIX );
    s.print ( lineBreak );
#else
    s.print ( F ( "Gateway type: MysGW" ) );
    s.print ( lineBreak );
    s.print ( F ( "network port: " ) );
    s.print ( MY_MYSGW_PORT );
    s.print ( lineBreak );
#endif
  s.print ( F ( "Radio type: " ) );
#ifdef MY_RADIO_RF24
    s.print ( F ( "NRF24" ) );
    s.print ( lineBreak );
#elif defined MY_RADIO_RFM69
    s.print ( F (  "RFM69" ) );
    s.print ( lineBreak );
    s.print ( F ( "network id: " ) );
    s.print ( MY_RFM69_NETWORKID );
    s.print ( lineBreak );
#else
    s.print ( F ( "bad radio defined" ) );
    s.print ( lineBreak );
#endif
  s.print ( F ( "signing: " ) );
#ifdef USE_SIGNING
    s.print ( "yes" );
#else
    s.print ( "no" );
#endif
  s.print ( lineBreak );
}

#include <Ethernet.h>
#include <MySensors.h>
#ifdef WEBFRONTEND
  byte mac[] = { MY_MAC_ADDRESS };
  IPAddress ip ( 192,168,99,13 );
  EthernetServer server ( 80 );
#endif

void setup() 
{
  uint32_t enter = hwMillis();
  while (hwMillis() - enter < (uint32_t)500);
  while(!MY_SERIALDEVICE); // For USB enabled devices, wait for USB enumeration before continuing
  printConfig( MY_SERIALDEVICE, "\n" );
#ifdef WEBFRONTEND
  Ethernet.begin(mac, ip);
#endif
}

void presentation() { }

void loop() 
{
#ifdef WEBFRONTEND
  // listen for incoming clients
  EthernetClient client = server.available();
  if (client) {
    Serial.println("new client");
    // an http request ends with a blank line
    boolean currentLineIsBlank = true;
    while (client.connected()) {
      if (client.available()) {
        char c = client.read();
        Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        if (c == '\n' && currentLineIsBlank) {
          // send a standard http response header
          client.println("HTTP/1.1 200 OK");
          client.println("Content-Type: text/html");
          client.println("Connection: close");  // the connection will be closed after completion of the response
          client.println();
          client.println("<!DOCTYPE HTML>");
          client.println("<html>");
          printConfig ( client, "<br />" );
          client.println("</html>");
          break;
        }
        if (c == '\n') {
          // you're starting a new line
          currentLineIsBlank = true;
        } else if (c != '\r') {
          // you've gotten a character on the current line
          currentLineIsBlank = false;
        }
      }
    }
    // give the web browser time to receive the data
    delay(1);
    // close the connection:
    client.stop();
    Serial.println("client disconnected");
  }
  #endif
}

It will deliver this website to the browser:

***********************************************
MySensors W5100 Gateway
Version 0.2
own mac address: DE:AD:BA:BE:99:13
own network address: 192.168.99.13
Gateway type: mqtt
mqtt controller address: 192.168.99.20
mqtt controller port: 1883
mqtt client id: mysgw1
mqtt publish prefix: mysgw1-out
mqtt subscribe prefix: mysgw1-in
Radio type: NRF24
signing: no

tssk

@frits Cool! So you already had this?

I was exactly thinking of using most simple HTTP server example from Arduino examples (although you have it nicely dynamic using the defines etc.). But I was not sure if it would not interfere with mysensors library in some way or if maybe mysensors already had such functionality build-in (because if I remember correctly Ethernet gateway return some data to browser of you call the 5003 port).

frits

@tssk said in 💬 Building a MQTT Gateway:

@frits Cool! So you already had this?

no, not as a webserver. I only print some node information to Serial.

Webserver seems to work and not to interfere with mysensors core. My (MQTT-)Gateway doesn't act as a server, so there's only one listener. But multiple listeners (webserver on port 80 and Ethernet Gateway on port 5003 for example) should also be possible. I've not tested this though.

linkinpio

Hello,
I'm new here so sorry if this is not the right place for this question.
But I am trying to build Arduino MQTT GW to act as a relay/light controller and report/interact with HomeAssistant.
I think I have worked out most of my problems and got it running as I wanted to, but I was testing failover senarios as the controller should be able to work independently of any other device as long it has power.
My config is kind of simple Arduino Mega with W5500 Ethernet module configured as MQTT GW
And the problem is that when network or HA/MQTT broker is unavailable when Arduino is restarted/booting it tries to reconnect to MQTT broker constantly and interupts the main loop everytime for around 3s which makes board basically unresponsive.

00:45:07.094 -> 549536 !GWT:RMQ:FAIL
00:45:07.094 -> Loop count = 179
00:45:07.094 -> 549538 GWT:TPC:IP=192.168.11.111
00:45:08.107 -> 550542 GWT:RMQ:CONNECTING...
00:45:10.130 -> 552547 !GWT:RMQ:FAIL
00:45:10.130 -> Loop count = 180
00:45:10.130 -> 552550 GWT:TPC:IP=192.168.11.111

Any ideas how I could disable reconnect function if that happens for example after 3 attempts? So the controller can work normally even if there is no connection to MQTT broker?

OldSurferDude

@linkinpio

I don't have a solution, but I, too, may run into that problem. So I'm curious how it can be resolved.

Perhaps post in the Hardware or Development sections. Fortunately, some really knowledgeable people (eg. @mfalkvidd and @NeverDie ) monitor pretty much all of the forum subjects.

OSD

electrik

@linkinpio in the pubsubclient library, you can change the timeout on connecting to the mqtt broker. Default it is 15s, reducing it to e.g. 2s will increase the availability of the main loop.

mfalkvidd

@OldSurferDude yes I have read it already. But unfortunately my impression is that my knowledge of how the code works is woefully insufficient.

linkinpio

@electrik thank you for your suggestion, but are you talking about MQTT_SOCKET_TIMEOUT ?
If yes I did set this to 1s

#define MQTT_SOCKET_TIMEOUT 1

and did not change a thing

21:32:05.081 -> Loop count = 153
21:32:05.081 -> 471258 GWT:TPC:IP=192.168.11.111
21:32:06.098 -> 472262 GWT:RMQ:CONNECTING...
21:32:08.071 -> 474266 !GWT:RMQ:FAIL
21:32:08.071 -> Loop count = 154

If not please do let me know what you had in mind so I can check it out.

@OldSurferDude thanks for confirming that I have not missed anything when doing a resarch on this matter, if there will be no new suggestions I think I might have to post it to Development as dont see anyone suggesting any good workaround or a solution.

electrik

No, you're right, it doesn't solve your issue. It does increase the availability of the main loop though.

linkinpio

@electrik
Have you tested this in your enviroment/sketch?
Did it increase availability for your sketch?
If yes what was the single loop time before and after changing this?
I did test is on my example and it does not increase the availability for the main loop for my sketch, also you can see the printout from the serial monitor, the loop is run in the same 3s intervals like before, so maybe I am doing something wrong?

electrik

Yes, that's what I meant. But the value seems to be 2 seconds in your log. Where have you placed the define?
You can also use

setSocketTimeout(1)

On the pubsubclient object.

linkinpio

@electrik I've put the

#define MQTT_SOCKET_TIMEOUT 1

in the beginning where all other #defined constants are and it does not seeams to take this config.
I tried to use

setSocketTimeout(1)

but I can't seems to get the sketch to work with <PubSubClient.h>
I've added

EthernetClient ethClient;
PubSubClient client(ethClient);
client.setSocketTimeout(1);

but getting following error.

GatewayW5x00MQTTClient.ino:172:1: error: 'client' does not name a type
 client.setSocketTimeout(1);
 ^~~~~~

exit status 1

Compilation error: 'client' does not name a type

any ideas?

electrik

@linkinpio said in 💬 Building a MQTT Gateway:

client.setSocketTimeout(1);

That is the right syntax, but you have to move the line in setup()

linkinpio

@electrik
tried it but also no luck

C:\Users\p.\Documents\Arduino\GatewayW5x00MQTTClient\GatewayW5x00MQTTClient.ino: In function 'void setup()':
C:\Users\p.\Documents\Arduino\GatewayW5x00MQTTClient\GatewayW5x00MQTTClient.ino:213:10: error: 'class PubSubClient' has no member named 'setSocketTimeout'
   client.setSocketTimeout(1);
          ^~~~~~~~~~~~~~~~

exit status 1

Compilation error: 'class PubSubClient' has no member named 'setSocketTimeout'```

electrik

This looks like you're not using the latest (development) version of the mysensors library. That has the latest pubsubclient library included.
Looking at the intervals between the errors again, I'm doubtful if it will make a big difference though. The mysensors library will need to be modified to fulfill your request I guess

linkinpio

I think I'm ussing the latest one

but as you said I don't think it will fix my issue though.
But looking at the examples I found that they have a connect function which then can be used to reconnect whenever needed not constantly like it happens now,

But not sure how I could integrate that functionality in this sketch though, as I'm not that experienced with programming :/
To you think it will be possible?

/*
 Reconnecting MQTT example - non-blocking

 This sketch demonstrates how to keep the client connected
 using a non-blocking reconnect function. If the client loses
 its connection, it attempts to reconnect every 5 seconds
 without blocking the main loop.

*/

#include <SPI.h>
#include <Ethernet.h>
#include <PubSubClient.h>

// Update these with values suitable for your hardware/network.
byte mac[]    = {  0xDE, 0xED, 0xBA, 0xFE, 0xFE, 0xED };
IPAddress ip(172, 16, 0, 100);
IPAddress server(172, 16, 0, 2);

void callback(char* topic, byte* payload, unsigned int length) {
  // handle message arrived
}

EthernetClient ethClient;
PubSubClient client(ethClient);

long lastReconnectAttempt = 0;

boolean reconnect() {
  if (client.connect("arduinoClient")) {
    // Once connected, publish an announcement...
    client.publish("outTopic","hello world");
    // ... and resubscribe
    client.subscribe("inTopic");
  }
  return client.connected();
}

void setup()
{
  client.setServer(server, 1883);
  client.setCallback(callback);

  Ethernet.begin(mac, ip);
  delay(1500);
  lastReconnectAttempt = 0;
}


void loop()
{
  if (!client.connected()) {
    long now = millis();
    if (now - lastReconnectAttempt > 5000) {
      lastReconnectAttempt = now;
      // Attempt to reconnect
      if (reconnect()) {
        lastReconnectAttempt = 0;
      }
    }
  } else {
    // Client connected

    client.loop();
  }

}

💬 Building a MQTT Gateway

16

11.7k

11.2k

113.0k