Pagination doesn't work with datatables and nodejs/elasticsearch implementation

Ghosty

I'm using nodejs as back/front-end implementation for a quick UI overview of an index in nodejs, and I'd like to use datatables with server side processing to display the data in a properly formated table.

It grabs all of the data fine, however it just dumps all of the records into one page on the table.

Model:

const { Client } = require('@elastic/elasticsearch')

const client = new Client({
    node: '',
    auth: {
        username: '',
        password: ''
    }
})

module.exports = function getElasticData(callback/*, size*/) {
    //parseInt(inputSize) = size;
    client.search({
        index: 'anomalies02_bhc-lstm2',
        size: 10000,
        body: {
            query: {
                match_all: {}
            }
        }
    }, function (error, response, status) {
        if (error) {
            console.log("search error: " + error)
        }
        if (response) {
            var elasticList = [];
            //console.log("EL:" + JSON.stringify(response))
            response.body.hits.hits.forEach(function (hit) {
                elasticList.push(hit);
            })
            //callback(elasticList.sort());
            //console.log("SORTED LIST: " + JSON.stringify(elasticList))
            callback(elasticList);
        }
        else {
            console.log("<p>No results</p>");
        }
    });
}

Controller:

var elasticDataModel = require('../model/getDataElastic');

exports.getData = function (req, res) {
    elasticDataModel(function (elasticList) {

        var searchStr = req.body.search.value;
        var recordsTotal = 0;
        var recordsFiltered = 0;

        var size = parseInt(req.body.length);
        var recordsFiltered = elasticList.slice(0, size)
        console.log("LENGTH: " + JSON.stringify(req.body.length))
        console.log("FILTERED: " + JSON.stringify(recordsFiltered))

        //console.log(elasticList[0]._source);
        console.log(typeof parseInt(req.body.draw))
        console.log(elasticList.length)
        recordsTotal = elasticList.length;
        //console.log("DRAW " + req.body.draw);
        var data = JSON.stringify({
            "data": elasticList,
            "draw": parseInt(req.body.draw),
            "recordsTotal": recordsTotal,
            "recordsFiltered": recordsFiltered
        });

        res.send(data);

    });

}

HTML:

        $(document).ready(function () {
            var t = $('#example2').DataTable({
                "paging": true,
                "processing": true,
                "serverSide": true,
                'ajax': {
                    'type': 'POST',
                    'url': '/populateData'
                },
                'pageLength': 20,
                'lengthMenu': [5, 10, 20, 50, 100, 200, 500],
                'columns':
                    [
                        { 'data': '_id', "defaultContent": "", 'name': 'ID' },
                        { "defaultContent": "", 'name': 'Kibana Link' },
                        { 'data': '_source.Environment', "defaultContent": "", 'name': 'Environment' },
                        { 'data': '_source.Cause', "defaultContent": "", 'name': 'Downtime cause' },
                        { 'data': '_source.Start', "defaultContent": "", 'name': 'Detected start' },
                        { 'data': '_source.End', "defaultContent": "", 'name': 'Detected end' },
                        { "defaultContent": "", 'name': 'Actual start' },
                        { "defaultContent": "", 'name': 'Actual end' },
                        { "defaultContent": "", 'name': 'Reason category' },
                        { "defaultContent": "", 'name': 'Reason details' },
                        { "defaultContent": "", 'name': 'Submit' },
                    ],
                "columnDefs": [
                    {
                        "searchable": true,
                        "orderable": true,
                        "targets": 0
                    }
                ]

            });

        });

kthorngren

I suspect your getElasticData function is not paging the results. Server side processing expects the server to perform the paging function. The SSP protocol is documented here. The server script is expected to use the parameters sent to limit the result, by using the start and length parameters, to the page being requested.

I suspect console.log(elasticList.length) shows a count of all the records which you are returning with "data": elasticList,. The number of records return should match 'pageLength': 20,.

Kevin

Ghosty

Thank you so much Kevin!

I modifed my code accordingly and now it works flawlessly - just need to work on the search function still.

Changed code:

Controller

var elasticDataModel = require('../model/getFilteredDataElastic');
var pageLength;
var pageStart;

exports.getData = function (req, res) {
    pageLength = req.body.length;
    pageStart = req.body.start
    console.log("MODEL Length: " + pageLength)
    elasticDataModel(pageLength, pageStart, function (elasticFilteredList) {

        var searchStr = req.body.search.value;
        var recordsTotal = 0;
        var recordsFiltered = 0;
        console.log("PAGE LENGTH: " + pageLength)
        if (req.body.search.value) {
            var regex = new RegExp(req.body.search.value, "i")
            searchStr = { $or: [{ '_id': regex }, { 'Environment': regex }, { 'Downtime cause': regex }, { 'Detected start': regex }, { 'Detected end': regex }, { 'Reason details': regex }] };
        }
        else {
            searchStr = {};
        }
        var size = parseInt(req.body.length);
        var recordsFiltered = elasticFilteredList

        var data = JSON.stringify({
            "data": elasticFilteredList,
            "draw": parseInt(req.body.draw),
            "recordsTotal": elasticFilteredList[0],
            "recordsFiltered": recordsFiltered
        });

        res.send(data);
    });

}

Model:

const { Client } = require('@elastic/elasticsearch')

const client = new Client({
    node: '',
    auth: {
        username: '',
        password: ''
    }
})

module.exports = function getElasticData(arg, arg2, callback) {
    console.log("CONTROLLER SIZE: " + arg)
    console.log("CONTROLLER FROM: " + arg2)

    client.search({
        index: 'anomalies02_bhc-lstm2',
        from: arg2,
        size: arg,
        body: {
            query: {
                match_all: {}
            }
        }
    }, function (error, response, status) {
        if (error) {
            console.log("search error: " + error)
        }
        if (response) {
            var elasticFilteredList = [];

            console.log("VALUE: " + response.body.hits.total.value);
            elasticFilteredList.push(response.body.hits.total.value);
            response.body.hits.hits.forEach(function (hit) {
                elasticFilteredList.push(hit);
            })
            callback(elasticFilteredList);
        }
        else {
            console.log("<p>No results</p>");
        }
    });
}