Serial Gateway w/ MQTT using nodeJS

[barduino]

Hi Folks

While adding mqtt feature to my controller and inspired by @cranky and his node-red controller, I've created a simple nodejs app to do 2 things:

• Get the information from the serial gateway and publish it to a broker
• Subscribe from the broker and send it to the serial gateway.

Here is the app.js

// LIBS
var com = require("serialport");
var mqtt = require('mqtt');

// SERIAL PORT SETTINGS
var serial_port = '/dev/ttyACM0';
// var serial_port = 'COM3'; // windows

// MQTT SETTINGS
var mqtt_host     = 'mqtts://m10.cloudmqtt.com';
var mqtt_port     =  8883;
var mqtt_username =  'user name';
var mqtt_password =  'password';

var KEY = 'yourkey';

// publications will be made into /haall/yourkey/out/x/x/x/x
// subcriptions will come from /haall/yourkey/in/x/x/x/
// adjust as necessary

// MQTT CLIENT OPTIONS
var mqtt_Options = {
     port:              mqtt_port
    ,keepalive:         10                  //seconds, set to 0 to disable
    ,clientId:          'haall_' + KEY
    ,protocolId:        'MQTT'
    //,protocolVersion: 4
    ,clean:             false               //set to false to receive QoS 1 and 2 messages while offline
    ,reconnectPeriod:   1000                // milliseconds, interval between two reconnections
    ,connectTimeout:    30 * 1000           //milliseconds, time to wait before a CONNACK is received
    ,username:          mqtt_username       //the username required by your broker, if any
    ,password:          mqtt_password       //the password required by your broker, if any
    /*
    ,incomingStore: , // a Store for the incoming packets
    ,outgoingStore: , // a Store for the outgoing packets
    */
    //a message that will sent by the broker automatically when the client disconnect badly. The format is:
    ,will: {topic:  '/haall/'+KEY+'/out',                           // the topic to publish
            payload:'Client haall_' + KEY +'has lost connection',   // the message to publish
            qos:    1,                                              // the QoS
            retain: false                                           // the retain flag 
    }    
}

// SERIAL PORT
var serialPort = new com.SerialPort(serial_port, {
    baudrate: 115200,
	parser: com.parsers.readline('\n')
  });

// START SERIAL PORT 
console.log('Opening Serial port...');
serialPort.open(function (error) {
  if ( error ) {
    console.log('failed to open: '+error);
  } else {
    console.log('Serial port opened!');
  }
});

// START MQTT
console.log('Starting MQTT...'); 
var mqtt_client  = mqtt.connect(mqtt_host,mqtt_Options);

console.log('Subscribing MQTT...'); 
mqtt_client.subscribe('/haall/'+KEY+'/in/#');

console.log('Publish MQTT...'); 
mqtt_client.publish('/haall/'+KEY+'/out', 'HAALL mqtt client started at '+ new Date());

// SERIAL PORT OUTGOING
serialPort.on('data', function(data) {
	
	if (data.indexOf('0;0;3;0;9;') == 0) {
		console.log('        I_LOG: '+data);
        mqttPublish(data);	
	}
	else{
		console.log(data);
		mqttPublish(data);
	}
});

// MQTT INCOMING
mqtt_client.on('message', function (topic, message) {
	var m = topic.toString();
	m = m.replace('/haall/'+KEY+'/in/','');
	m = m.split('/').join(';');
	m = m + ';' + message.toString();
	serialPort.write('' + m + '\n');

  	console.log('INCOMING MQTT: ' + topic + ':' + message.toString());
});

// MQTT OUTGOING
function mqttPublish(data){
	var topic = '/haall/'+KEY+'/out/';
	var params = data.split(';');
	topic = topic + params[0] + '/';
	topic = topic + params[1] + '/';
	topic = topic + params[2] + '/';
	topic = topic + params[3] + '/';
	topic = topic + params[4] + '';
	var payload = params[5];
	mqtt_client.publish(topic,payload);
	console.log('OUTGOING MQTT: ' + topic + ' Payload: ' + payload);
}

and the package.json

{
  "name": "gw",
  "version": "0.0.2",
  "description": "Gateway Module for MySensors",
  "dependencies": {
    "serialport": "^1.4.5",
    "mqtt": "^1.7.0"
  },
  "main": "gw.js",
  "devDependencies": {},
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC"
}

Cheers

[icebob]

@barduino Thx, good job!

[รอเรือ]

@barduino said in Serial Gateway w/ MQTT using nodeJS:

Hi Folks

While adding mqtt feature to my controller and inspired by @cranky and his node-red controller, I've created a simple nodejs app to do 2 things:

• Get the information from the serial gateway and publish it to a broker
• Subscribe from the broker and send it to the serial gateway.

Here is the app.js

// LIBS
var com = require("serialport");
var mqtt = require('mqtt');

// SERIAL PORT SETTINGS
var serial_port = '/dev/ttyACM0';
// var serial_port = 'COM3'; // windows

// MQTT SETTINGS
var mqtt_host     = 'mqtts://m10.cloudmqtt.com';
var mqtt_port     =  8883;
var mqtt_username =  'user name';
var mqtt_password =  'password';

var KEY = 'yourkey';

// publications will be made into /haall/yourkey/out/x/x/x/x
// subcriptions will come from /haall/yourkey/in/x/x/x/
// adjust as necessary

// MQTT CLIENT OPTIONS
var mqtt_Options = {
     port:              mqtt_port
    ,keepalive:         10                  //seconds, set to 0 to disable
    ,clientId:          'haall_' + KEY
    ,protocolId:        'MQTT'
    //,protocolVersion: 4
    ,clean:             false               //set to false to receive QoS 1 and 2 messages while offline
    ,reconnectPeriod:   1000                // milliseconds, interval between two reconnections
    ,connectTimeout:    30 * 1000           //milliseconds, time to wait before a CONNACK is received
    ,username:          mqtt_username       //the username required by your broker, if any
    ,password:          mqtt_password       //the password required by your broker, if any
    /*
    ,incomingStore: , // a Store for the incoming packets
    ,outgoingStore: , // a Store for the outgoing packets
    */
    //a message that will sent by the broker automatically when the client disconnect badly. The format is:
    ,will: {topic:  '/haall/'+KEY+'/out',                           // the topic to publish
            payload:'Client haall_' + KEY +'has lost connection',   // the message to publish
            qos:    1,                                              // the QoS
            retain: false                                           // the retain flag 
    }    
}

// SERIAL PORT
var serialPort = new com.SerialPort(serial_port, {
    baudrate: 115200,
	parser: com.parsers.readline('\n')
  });

// START SERIAL PORT 
console.log('Opening Serial port...');
serialPort.open(function (error) {
  if ( error ) {
    console.log('failed to open: '+error);
  } else {
    console.log('Serial port opened!');
  }
});

// START MQTT
console.log('Starting MQTT...'); 
var mqtt_client  = mqtt.connect(mqtt_host,mqtt_Options);

console.log('Subscribing MQTT...'); 
mqtt_client.subscribe('/haall/'+KEY+'/in/#');

console.log('Publish MQTT...'); 
mqtt_client.publish('/haall/'+KEY+'/out', 'HAALL mqtt client started at '+ new Date());

// SERIAL PORT OUTGOING
serialPort.on('data', function(data) {
	
	if (data.indexOf('0;0;3;0;9;') == 0) {
		console.log('        I_LOG: '+data);
        mqttPublish(data);	
	}
	else{
		console.log(data);
		mqttPublish(data);
	}
});

// MQTT INCOMING
mqtt_client.on('message', function (topic, message) {
	var m = topic.toString();
	m = m.replace('/haall/'+KEY+'/in/','');
	m = m.split('/').join(';');
	m = m + ';' + message.toString();
	serialPort.write('' + m + '\n');

  	console.log('INCOMING MQTT: ' + topic + ':' + message.toString());
});

// MQTT OUTGOING
function mqttPublish(data){
	var topic = '/haall/'+KEY+'/out/';
	var params = data.split(';');
	topic = topic + params[0] + '/';
	topic = topic + params[1] + '/';
	topic = topic + params[2] + '/';
	topic = topic + params[3] + '/';
	topic = topic + params[4] + '';
	var payload = params[5];
	mqtt_client.publish(topic,payload);
	console.log('OUTGOING MQTT: ' + topic + ' Payload: ' + payload);
}

and the package.json

{
  "name": "gw",
  "version": "0.0.2",
  "description": "Gateway Module for MySensors",
  "dependencies": {
    "serialport": "^1.4.5",
    "mqtt": "^1.7.0"
  },
  "main": "gw.js",
  "devDependencies": {},
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC"
}

Cheers

@barduino, this is great stuff, any chance that there is an updated version of app.js? I've been trying hard to get this working.

Cheers!

[รอเรือ]

Anyway, I put some effort in this because I think that the Serial-to-MQTT gateway is actually the ultimate gateway for MySensors. It's very easy to build and it's versatile because it uses MQTT which in many cases will eliminate the use of more or less buggy add-ons for different home automation systems. It's my hope that the team would like to consider to add it as a new gateway type.

I used the code from @barduino as a starting point but there has been many changes to it. Here it goes:

/* Serial Gateway w/ MQTT using nodeJS, originally written by barduino https://forum.mysensors.org/user/barduino
 * Rewritten by รอเรือ
 *
 * Copyright: Unknown
 *
 * MySensors forum topic: https://forum.mysensors.org/topic/3042/serial-gateway-w-mqtt-using-nodejs
 *
 * Using:
 *		serialport -npm: https://www.npmjs.com/package/serialport
 *		mqtt -npm: https://www.npmjs.com/package/mqtt#client
 */

// Custom settings
var serial_port = '/dev/ttyUSBmys';
var serial_baud_rate = 115200;
var mqtt_host     = 'XXXXXXXX';
var mqtt_port     =  1883;
var mqtt_subscribe_topic = 'mysensors-in';
var mqtt_publish_topic = 'mysensors-out';
var mqtt_username =  '';
var mqtt_password =  '';
var mqtt_client_id = 'mysensors_' + Math.random().toString(16).substr(2, 8);
const DEBUG = false;

// LIBS
const SerialPort = require('serialport');
const Readline = require('@serialport/parser-readline')
const mqtt = require('mqtt');

// MQTT CLIENT OPTIONS
var mqtt_Options = {
	port: mqtt_port,
	keepalive: 60, //seconds, set to 0 to disable
	clientId: mqtt_client_id,
	protocolId: 'MQTT', //'MQTT', 'mqtts', 'tcp', 'tls', 'ws', 'wss'. 
	//protocolVersion: 4,
	clean: true, //set to false to receive QoS 1 and 2 messages while offline
	reconnectPeriod: 5000, // milliseconds, interval between two reconnections
	connectTimeout: 30 * 1000, //milliseconds, time to wait before a CONNACK is received
	//username: mqtt_username, //the username required by your broker, if any
	//password: mqtt_password, //the password required by your broker, if any
	will: {topic: mqtt_publish_topic, payload: mqtt_client_id +' disconnected', qos: 1, retain: false }
}

// Define the port and a parser
const port = new SerialPort(serial_port, { baudRate: serial_baud_rate });
const parser = port.pipe(new Readline({ delimiter: '\n' }))

// START MQTT
if (DEBUG) console.log('Starting MQTT...');
var mqtt_client = mqtt.connect('mqtt://'+mqtt_host, mqtt_Options);

mqtt_client.on('connect', function () {
	if (DEBUG) console.log('Subscribing MQTT...');
	mqtt_client.subscribe(mqtt_subscribe_topic+'/#');

	if (DEBUG) console.log('Publish MQTT...'); 
	mqtt_client.publish(mqtt_publish_topic, 'mqtt client '+mqtt_client_id+' started at '+ new Date());
})

parser.on('data', function (data) {
	if (DEBUG) console.log('Data:', data);
	mqttPublish(data);
});

// MQTT INCOMING
mqtt_client.on('message', function (topic, message) {
	var m = topic.toString();
	m = m.replace(mqtt_subscribe_topic, '');
	m = m.split('/').join(';');
	m = m + ';' + message.toString();
	port.write('' + m + '\n', function(err) {
		if (err) {
		  return console.log('Error on write: ', err.message);
		}
		if (DEBUG) console.log('INCOMING MQTT: ' + topic + ':' + message.toString());
	});
});

// MQTT OUTGOING
function mqttPublish(data){
	var topic = mqtt_publish_topic+'/';
	var params = data.split(';');
	topic = topic + params[0] + '/';
	topic = topic + params[1] + '/';
	topic = topic + params[2] + '/';
	topic = topic + params[3] + '/';
	topic = topic + params[4] + '';
	var payload = params[5];
	mqtt_client.publish(topic,payload);
	if (DEBUG) console.log('OUTGOING MQTT: ' + topic + ' ' + payload);
}

Use pm2 to keep it running!

Cheers!!

Very simple hardware:

0_1533390263234_DSC_1915.JPG

[electrik]

nice, thanks for sharing
Could you also explain how to install / run this?

[รอเรือ]

@electrik

There are already plenty of instructions on how you install Node.js and it would be beyond the scope of this topic to do so here. There are so many variables such as operating systems and so on to take into account.

After installing Node.js you need to install the Node serialport and Node mqtt.

pm2 is a very nice process manager for Node.js that you can use to make sure that your node scripts always is running. It's highly recommended and will make your life easier.

In my own installation, I created a sub directory ~/node_modules/serialgwmqtt/ where I saved the app.js script file. Then cd ~/node_modules/serialgwmqtt/ followed by node app.js to make sure that everything works fine.

When it does (works fine) you should make it autostart. I added my script to pm2 with the following command: pm2 start app.js --name "serialgwmqtt" (while still in the same directory). check hat it's working by issuing pm2 list

I might have been running npm installor something similar also. I'm actually not so clever with these things and I'm sure my instructions can be simplified. The script should also be uploaded so some place to make it available for installing with a simple command. Maybe someone else more node.js-talented person want's to help here.

I hope I have't forgotten anything, it's quite straight forward.

Things are running rock solid here.