MY_GATEWAY_TINYGSM

[Thucar]

It is alive!!!

MQTT Client gateway with GSM modems and ESP8266 as a modem seems to be working as intended.

To replicate, do this:

• Install the TinyGSM library
• Use this MyGatewayTransportMQTTClient.cpp

/*
* The MySensors Arduino library handles the wireless radio link and protocol
* between your home built sensors/actuators and HA controller of choice.
* The sensors forms a self healing radio network with optional repeaters. Each
* repeater and gateway builds a routing tables in EEPROM which keeps track of the
* network topology allowing messages to be routed to nodes.
*
* Created by Henrik Ekblad <henrik.ekblad@mysensors.org>
* Copyright (C) 2013-2017 Sensnology AB
* Full contributor list: https://github.com/mysensors/Arduino/graphs/contributors
*
* Documentation: http://www.mysensors.org
* Support Forum: http://forum.mysensors.org
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* version 2 as published by the Free Software Foundation.
*/


// Topic structure: MY_MQTT_PUBLISH_TOPIC_PREFIX/NODE-ID/SENSOR-ID/CMD-TYPE/ACK-FLAG/SUB-TYPE

#include "MyGatewayTransport.h"

#if defined MY_CONTROLLER_IP_ADDRESS
IPAddress _brokerIp(MY_CONTROLLER_IP_ADDRESS);
#endif

#if defined(MY_IP_ADDRESS)
IPAddress _MQTT_clientIp(MY_IP_ADDRESS);
#if defined(MY_IP_GATEWAY_ADDRESS)
IPAddress _gatewayIp(MY_IP_GATEWAY_ADDRESS);
#elif defined(MY_GATEWAY_ESP8266) /* Elif part of MY_IP_GATEWAY_ADDRESS */
// Assume the gateway will be the machine on the same network as the local IP
// but with last octet being '1'
IPAddress _gatewayIp(_MQTT_clientIp[0], _MQTT_clientIp[1], _MQTT_clientIp[2], 1);
#endif /* End of MY_IP_GATEWAY_ADDRESS */
#if defined(MY_IP_SUBNET_ADDRESS)
IPAddress _subnetIp(MY_IP_SUBNET_ADDRESS);
#elif defined(MY_GATEWAY_ESP8266) /* Elif part of MY_IP_SUBNET_ADDRESS */
IPAddress _subnetIp(255, 255, 255, 0);
#endif /* End of MY_IP_SUBNET_ADDRESS */
#endif /* End of MY_IP_ADDRESS */

#if defined(MY_GATEWAY_ESP8266)
#define EthernetClient WiFiClient
#elif defined(MY_GATEWAY_LINUX)
// Nothing to do here
#else
uint8_t _MQTT_clientMAC[] = { MY_MAC_ADDRESS };
#endif /* End of MY_GATEWAY_ESP8266 */

#ifdef MY_GATEWAY_TINYGSM
#include <TinyGsmClient.h>
static TinyGsm modem(SerialAT);
static TinyGsmClient _MQTT_gsmClient(modem);
static PubSubClient _MQTT_client(_MQTT_gsmClient);
#if defined(MY_GSM_BAUDRATE)
uint32_t rate = MY_GSM_BAUDRATE;
#else
uint32_t rate = 0;
#endif
#else
static EthernetClient _MQTT_ethClient;
static PubSubClient _MQTT_client(_MQTT_ethClient);
#endif /* End of MY_GATEWAY_TINYGSM */


static bool _MQTT_connecting = true;
static bool _MQTT_available = false;
static MyMessage _MQTT_msg;

bool gatewayTransportSend(MyMessage &message)
{
	if (!_MQTT_client.connected()) {
		return false;
	}
	setIndication(INDICATION_GW_TX);
	char *topic = protocolFormatMQTTTopic(MY_MQTT_PUBLISH_TOPIC_PREFIX, message);
	GATEWAY_DEBUG(PSTR("GWT:TPS:TOPIC=%s,MSG SENT\n"), topic);
#if defined(MY_MQTT_CLIENT_PUBLISH_RETAIN)
	bool retain = mGetCommand(message) == C_SET ||
	              (mGetCommand(message) == C_INTERNAL && message.type == I_BATTERY_LEVEL);
#else
	bool retain = false;
#endif /* End of MY_MQTT_CLIENT_PUBLISH_RETAIN */
	return _MQTT_client.publish(topic, message.getString(_convBuffer), retain);
}

void incomingMQTT(char* topic, uint8_t* payload, unsigned int length)
{
	GATEWAY_DEBUG(PSTR("GWT:IMQ:TOPIC=%s, MSG RECEIVED\n"), topic);
	_MQTT_available = protocolMQTTParse(_MQTT_msg, topic, payload, length);
}

bool reconnectMQTT(void)
{
	GATEWAY_DEBUG(PSTR("GWT:RMQ:MQTT RECONNECT\n"));
	// Attempt to connect
	if (_MQTT_client.connect(MY_MQTT_CLIENT_ID
#if defined(MY_MQTT_USER) && defined(MY_MQTT_PASSWORD)
	                         , MY_MQTT_USER, MY_MQTT_PASSWORD
#endif
	                        )) {
		GATEWAY_DEBUG(PSTR("GWT:RMQ:MQTT CONNECTED\n"));

		// Send presentation of locally attached sensors (and node if applicable)
		presentNode();

		// Once connected, publish an announcement...
		//_MQTT_client.publish("outTopic","hello world");
		// ... and resubscribe
		_MQTT_client.subscribe(MY_MQTT_SUBSCRIBE_TOPIC_PREFIX "/+/+/+/+/+");
		return true;
	}
	return false;
}

bool gatewayTransportConnect(void)
{
#if defined(MY_GATEWAY_ESP8266)
	while (WiFi.status() != WL_CONNECTED) {
		wait(500);
		GATEWAY_DEBUG(PSTR("GWT:TPC:CONNECTING...\n"));
	}
	GATEWAY_DEBUG(PSTR("GWT:TPC:IP=%s\n"),WiFi.localIP().toString().c_str());
#elif defined(MY_GATEWAY_LINUX) /* Elif part of MY_GATEWAY_ESP8266 */
#if defined(MY_IP_ADDRESS)
	_MQTT_ethClient.bind(_MQTT_clientIp);
#endif /* End of MY_IP_ADDRESS */
#elif defined(MY_GATEWAY_TINYGSM)
    GATEWAY_DEBUG(PSTR("GWT:TPC:IP=%s\n"), modem.getLocalIP().c_str());
#else /* Else part of MY_GATEWAY_ESP8266 */
#if defined(MY_IP_ADDRESS)
	Ethernet.begin(_MQTT_clientMAC, _MQTT_clientIp);
#else /* Else part of MY_IP_ADDRESS */
	// Get IP address from DHCP
	if (!Ethernet.begin(_MQTT_clientMAC)) {
		GATEWAY_DEBUG(PSTR("!GWT:TPC:DHCP FAIL\n"));
		_MQTT_connecting = false;
		return false;
	}
#endif /* End of MY_IP_ADDRESS */
	GATEWAY_DEBUG(PSTR("GWT:TPC:IP=%" PRIu8 ".%" PRIu8 ".%" PRIu8 ".%" PRIu8 "\n"),
	              Ethernet.localIP()[0],
	              Ethernet.localIP()[1], Ethernet.localIP()[2], Ethernet.localIP()[3]);
	// give the Ethernet interface a second to initialize
	delay(1000);
#endif /* End of MY_GATEWAY_ESP8266 */
	return true;
}

bool gatewayTransportInit(void)
{
	_MQTT_connecting = true;

#if defined(MY_GATEWAY_TINYGSM)

#if defined(MY_GSM_RX) && defined(MY_GSM_TX)
	SoftwareSerial SerialAT(MY_GSM_RX, MY_GSM_TX);
#else
	// TODO: Needs sanity checks
#endif

#if !defined(MY_GSM_BAUDRATE)
	rate = TinyGsmAutoBaud(SerialAT);
#endif

	SerialAT.begin(rate);
	delay(3000);

	modem.restart();

#if defined(MY_GSM_PIN) && !defined(TINY_GSM_MODEM_ESP8266)
	modem.simUnlock(MY_GSM_PIN);
#endif

#ifndef TINY_GSM_MODEM_ESP8266
	if (!modem.waitForNetwork()) {
		GATEWAY_DEBUG(PSTR("!GWT:TIN:GPRS FAIL\n"));
		while (true);
	}
	GATEWAY_DEBUG(PSTR("GWT:TIN:GPRS OK\n"));

	if (!modem.gprsConnect(MY_GSM_APN, MY_GSM_USR, MY_GSM_PSW)) {
		GATEWAY_DEBUG(PSTR("!GWT:TIN:APN FAIL\n"));
		while (true);
	}
	GATEWAY_DEBUG(PSTR("GWT:TIN:APN OK\n"));
	delay(1000);
#else
    if (!modem.networkConnect(MY_GSM_SSID, MY_GSM_PSW)) {
        GATEWAY_DEBUG(PSTR("!GWT:TIN:WIFI AP FAIL\n"));
        while (true);
      }
      GATEWAY_DEBUG(PSTR("GWT:TIN:WIFI AP OK\n"));
      delay(1000);
#endif

#endif /* End of MY_GATEWAY_TINYGSM */

#if defined(MY_CONTROLLER_IP_ADDRESS)
	_MQTT_client.setServer(_brokerIp, MY_PORT);
#else
	_MQTT_client.setServer(MY_CONTROLLER_URL_ADDRESS, MY_PORT);
#endif /* End of MY_CONTROLLER_IP_ADDRESS */

	_MQTT_client.setCallback(incomingMQTT);

#if defined(MY_GATEWAY_ESP8266)
	// Turn off access point
	WiFi.mode(WIFI_STA);
#if defined(MY_ESP8266_HOSTNAME)
	WiFi.hostname(MY_ESP8266_HOSTNAME);
#endif /* End of MY_ESP8266_HOSTNAME */
#if defined(MY_IP_ADDRESS)
	WiFi.config(_MQTT_clientIp, _gatewayIp, _subnetIp);
#endif /* End of MY_IP_ADDRESS */
#ifndef MY_ESP8266_BSSID
#define MY_ESP8266_BSSID NULL
#endif
	(void)WiFi.begin(MY_ESP8266_SSID, MY_ESP8266_PASSWORD, 0, MY_ESP8266_BSSID);
#endif /* End of MY_GATEWAY_ESP8266 */

	gatewayTransportConnect();

	_MQTT_connecting = false;
	return true;
}

bool gatewayTransportAvailable(void)
{
	if (_MQTT_connecting) {
		return false;
	}
	//keep lease on dhcp address
	//Ethernet.maintain();
	if (!_MQTT_client.connected()) {
		//reinitialise client
		if (gatewayTransportConnect()) {
			reconnectMQTT();
		}
		return false;
	}
	_MQTT_client.loop();
	return _MQTT_available;
}

MyMessage & gatewayTransportReceive(void)
{
	// Return the last parsed message
	_MQTT_available = false;
	return _MQTT_msg;
}

• Define these in your gateway sketch:

#define MY_GATEWAY_MQTT_CLIENT
// Enable gateway ethernet module type
#define MY_GATEWAY_TINYGSM

// Define the GSM modem. See TinyGSM documentation for full list
//#define TINY_GSM_MODEM_ESP8266
#define TINY_GSM_MODEM_A6
#define MY_GSM_APN "internet" // your mobile providers APN
#define MY_GSM_USR "" // Leave empty if not used
#define MY_GSM_PIN "" // Leave empty if not used
#define MY_GSM_PSW "" // This is either your mobile providers password or WiFi password, depending if you are using the ESP8266 or a GSM modem
#define MY_GSM_SSID ""

// Use Hardware Serial on Mega, Leonardo, Micro
#define SerialAT Serial1
#define MQTT_VERSION MQTT_VERSION_3_1 // I apparently need this to be able to run mosquitto

[gohan]

@thucar said in MY_GATEWAY_TINYGSM:

GSM modems and ESP8266 as a modem

Did you mean esp8266 as gateway or...? BTW great job :+1:

[Thucar]

@gohan said in MY_GATEWAY_TINYGSM:

@thucar said in MY_GATEWAY_TINYGSM:

GSM modems and ESP8266 as a modem

Did you mean esp8266 as gateway or...? BTW great job :+1:

Nope, ESP8266 is a modem in this scenario. So in my case I'm using an Arduino Mega 2560 with loads of IOs as the gateway and the ESP8266 as a Wifi modem for connectivity. No sketch is uploaded on the ESP and the only connections used (in addition to power and ground of course) are TX/RX from ESP to the Arduino.

[astron]

Hello.
Is it possible to add support for the gateway with TinyGSM without MQTT?
Controller IoBroker can work in server mode for a gateway with TinyGSM.

[jkandasa]

@thucar Great work! Thank you!
I am using MY_GATEWAY_TINYGSM from a month ago. I face Gateway hangs after a week and it needs a hard reboot(Power OFF and ON). Are you facing any issue similar to this?

My Hardware details:

• ESP8266 (ESP-12E)
• RFM69HW
• SIM800L

To power up GSM modem I am using 1500mA 18650 battery with charger board.

Can you share your hardware details?

[jkandasa]

After some days, internet connection disconnected(identified via SIM800L LED blink status), but still, ESP says all ok. I lost communication from controller too.

193617834 RFM69:SAC:SEND ACK,TO=11,RSSI=-57
193617847 RFM69:CSMA:RSSI=-107
193617866 TSF:MSG:READ,12-11-0,s=255,c=3,t=0,pt=1,l=1,sg=0:48
193617882 GWT:TPS:TOPIC=out_rfm69/12/255/3/0/0,MSG SENT
193629390 RFM69:SAC:SEND ACK,TO=1,RSSI=-82
193629403 RFM69:CSMA:RSSI=-107
193629421 TSF:MSG:READ,1-1-0,s=1,c=1,t=25,pt=1,l=1,sg=0:64
193629437 GWT:TPS:TOPIC=out_rfm69/1/1/1/0/25,MSG SENT
193629735 RFM69:SAC:SEND ACK,TO=1,RSSI=-49
193629747 RFM69:CSMA:RSSI=-106
193629766 TSF:MSG:READ,1-1-0,s=1,c=1,t=35,pt=7,l=5,sg=0:70.60
193629782 GWT:TPS:TOPIC=out_rfm69/1/1/1/0/35,MSG SENT
193650334 RFM69:SAC:SEND ACK,TO=11,RSSI=-98
193650347 RFM69:CSMA:RSSI=-103
193650366 TSF:MSG:READ,11-11-0,s=1,c=1,t=0,pt=7,l=5,sg=0:30.75
193650382 GWT:TPS:TOPIC=out_rfm69/11/1/1/0/0,MSG SENT
193659264 RFM69:SAC:SEND ACK,TO=1,RSSI=-102
193659277 RFM69:CSMA:RSSI=-106
193659296 TSF:MSG:READ,1-1-0,s=1,c=1,t=25,pt=1,l=1,sg=0:64
193659311 GWT:TPS:TOPIC=out_rfm69/1/1/1/0/25,MSG SENT
193659609 RFM69:SAC:SEND ACK,TO=1,RSSI=-92
193659621 RFM69:CSMA:RSSI=-108
193659640 TSF:MSG:READ,1-1-0,s=1,c=1,t=35,pt=7,l=5,sg=0:70.60
193659656 GWT:TPS:TOPIC=out_rfm69/1/1/1/0/35,MSG SENT
193673497 RFM69:SAC:SEND ACK,TO=11,RSSI=-93
193673510 RFM69:CSMA:RSSI=-103
193673529 TSF:MSG:READ,12-11-0,s=1,c=1,t=38,pt=7,l=5,sg=0:12.89
193673546 GWT:TPS:TOPIC=out_rfm69/12/1/1/0/38,MSG SENT
193674042 RFM69:SAC:SEND ACK,TO=11,RSSI=-99
193674054 RFM69:CSMA:RSSI=-103
193674073 TSF:MSG:READ,12-11-0,s=1,c=1,t=0,pt=7,l=5,sg=0:27.00
193674090 GWT:TPS:TOPIC=out_rfm69/12/1/1/0/0,MSG SENT
193674585 RFM69:SAC:SEND ACK,TO=11,RSSI=-99
193674598 RFM69:CSMA:RSSI=-105
193674617 TSF:MSG:READ,12-11-0,s=1,c=1,t=25,pt=1,l=1,sg=0:247
193674633 GWT:TPS:TOPIC=out_rfm69/12/1/1/0/25,MSG SENT
193675129 RFM69:SAC:SEND ACK,TO=11,RSSI=-105
193675142 RFM69:CSMA:RSSI=-108
193675160 TSF:MSG:READ,12-11-0,s=3,c=1,t=38,pt=7,l=5,sg=0:13.65
193675177 GWT:TPS:TOPIC=out_rfm69/12/3/1/0/38,MSG SENT
193675675 RFM69:SAC:SEND ACK,TO=11,RSSI=-99
193675688 RFM69:CSMA:RSSI=-102
193675707 TSF:MSG:READ,12-11-0,s=3,c=1,t=39,pt=7,l=5,sg=0:0.97
193675723 GWT:TPS:TOPIC=out_rfm69/12/3/1/0/39,MSG SENT
193676219 RFM69:SAC:SEND ACK,TO=11,RSSI=-103
193676232 RFM69:CSMA:RSSI=-106
193676251 TSF:MSG:READ,12-11-0,s=0,c=1,t=38,pt=7,l=5,sg=0:4.979
193676268 GWT:TPS:TOPIC=out_rfm69/12/0/1/0/38,MSG SENT
193676755 RFM69:SAC:SEND ACK,TO=11,RSSI=-97

I did a hard reboot of ESP8266, all back to normal.

[astron]

@jkandasa
I'm just designing a smart home system.
Do you use a MQTT connection to the controller?

ESP8266 memory is enough. You can add a communication check to the controller (for example, a request for the current time). And then you can restart SIM800L through an external pin on the module, and you can also restart ESSP8266.

[alexsh1]

Is there a chance to reboot GSM modem / gateway on a regular basis?
What I have discovered is that some operators cells stop communicating with the modem if the location is not changed for days. Therefore, modem has to be rebooted manually or on the timer, i.e. rebooting every day at 04:00 morning time.

[mfalkvidd]

@alexsh1 calling gatewayTransportInit() might work.

[shajek]

@mfalkvidd can you descirbe how ? where to place to periodically check if connected, and if not do some action ?

[mfalkvidd]

@shajek sorry I don't know

[alowhum]

@alexsh1

Here's a bit of code I use to send an SMS once in a while. Providers will disable your simcard if you don't do anything that makes them money once in a while. Perhaps you can use it to reset.

#ifdef SEND_SMS_EVERY_49_DAYS
  // This is an optional feature.
  // Every 49,7 days (using the millis() rollover) the system sends out an SMS. This helps keep the simcard active and registered on the GSM network. 
  // Use at your own risk: if the system experiences a power loss, the timer starts at 0 again. If your experience frequent powerlosses, then the simcard might be-deregistered anyway, since the keep-alive SMS's won't get sent. 
  // A slight delay is built in: the first sms is sent a week after the smart lock becomes active. This avoid sending a lot of SMS's if you are still playing with setting up the device, and powerlosses may be frequent.
  // This smart lock also offers another option. You can let the controller trigger the sending of the sms using the 'send test sms' button. Of course, your controller could also be down when you scheduled to trigger the button.
  static bool keepAliveSMSsent = false;             // used to make sure an SMS is only sent as the milliseconds starts, and not during every loop in the millisecond.
  if(millis() < 5){
    keepAliveSMSsent  = false;                      // when the clock rolls over, set the variable back so that a new SMS can be sent.
  }
  if (millis() > 604800000 && millis() < 604800010 && keepAliveSMSsent == false){ // 604800000 = 1 week in milliseconds. Sending the first keep-alive SMS after a week avoids sending a lot of SMS-es while testing the system (which may involve a lot of reboots).
    keepAliveSMSsent  = true;
    sendStatusSMS();
  }
#endif

Alternatively, you could check out the smart alarm clock code. It will show you how to request the time from the controller, turn that into a human readable time, and then you can do your thing.

uint32_t unixTime = 0;


void receiveTime(unsigned long controllerTime) {
  Serial.print(F("Received time: ")); Serial.println(controllerTime);
  unixTime = controllerTime;
  breakUpTime(unixTime);
}



void breakUpTime(uint32_t timeInput)
{
// Break the given time_t into time components.
// This is a more compact version of the C library localtime function
// Note that year is offset from 1970!

  uint32_t time;

  time = (uint32_t)timeInput;
  uint32_t Second = time % 60;
  time /= 60; // now it is minutes
  minutes = time % 60;
  time /= 60; // now it is hours
  hours = time % 24;
  time /= 24; // now it is days
  //int Wday = ((time + 4) % 7) + 1;                  // Which day of the week is it. Sunday is day 1 
  
  Serial.print(F("Calculated time: "));
  Serial.print(hours);
  Serial.print(F(":"));
  Serial.println(minutes);
}