Only 100 items returned?

[tb01923]

I have the following app that I want to use to grab a bunch of fields from vSphere and stick into a monog repository (which I am then cross reference to chef data and nmap scans to provide visibiliy inot network configuration and orchestration).

The relevant portions are the connect and getVirtualMachines methods. The first is pretty straight forward and returns a Task type resolving to (on success) the VC library instance. I would love the second method getVirtualMachines to return a Task resolving to (on success) all the virtual machines managed by the vcenter instance I am connected to. I am managing 104 virtual machiens in this instance - yet the request only returns an array of 100.

Is this a bug - OR am I not utiuilizing the library correctly?

const process = require('process')
    , mongoose = require('mongoose')
    , Task = require('data.task')
    , Async = require('control.async')(Task)
    , iprange = require('iprange')

const R = require('ramda')

const vSphere = require('../schemas/virtual-machines').model
    , adapt = require('../data-access/mongoose-data.task-adapters.js')

const tap = function(x){
    return x
}

// vSphereUpsert :: (criteria, {}) => Task {ok :: int, ...}
const vSphereUpsert = adapt.findOneAndUpdate(vSphere.collection)

// persist :: [{address :: string, ...}] => [Task {ok :: int, ...}]
const persist = R.map((data) => {
    return vSphereUpsert({ "name": data.name}, data)
})

const connect = () => {
    return new Task( (reject, resolve) => {
        const Vsphere = require('vsphere');
        const vc = new Vsphere.Client('10.10.10.10', 'domain\\user', 'password', false);

        vc.once('ready', () => resolve(vc));
        vc.once('error', reject);
    })
}

const getVirtualMachines = (vc) => {
    return new Task( (reject, resolve) => {
        const rootFolder = vc.serviceContent.rootFolder;
        const vms =  vc.getMORefsInContainerByType( rootFolder, 'VirtualMachine');
        vms.once('result', resolve)
        vms.once('error', reject)
    })
}

const findByName = (name) => R.filter(R.propEq('name', name))

const pluckVmSummary = R.pipe(
    R.prop('returnval'),
    R.prop('objects'),
    R.pluck('propSet'),
    R.map(findByName('summary')),
    R.flatten(),
    R.map(R.prop('val'))
)

const transform = R.map(vm =>{
    const retval = {}
    retval.address = vm.guest.ipAddress
    retval.name = vm.guest.hostName || vm.config.name
    retval.isPoweredOn = vm.runtime.powerState === 'poweredOn'
    retval.memory = parseInt(vm.config.memorySizeMB)
    retval.numCpu = parseInt(vm.config.numCpu)
    retval.numNics = parseInt(vm.config.numEthernetCards)
    retval.imageType = vm.guest.guestFullName || vm.config.guestFullName
    return retval
})

module.exports.scan = function(e,f){
    process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
    mongoose.connect('mongodb://localhost/network');

    connect().
        chain(getVirtualMachines).
        map(tap).
        map(pluckVmSummary).
        map(transform).
        chain(R.compose(Async.parallel, persist)).     // Task [ {ok :: int}]
        map(R.partition(R.propEq('ok', 1))).                    // Task [ [{ok : 1}], [{}]]
        fork(e,f )
}

module.exports.scan(
    console.error,
    console.log
)

[reedog117]

It seems that this is actually a limitation of the vSphere API. According to this link on the VMware Communities forum, people seem to be having similar issues.

Just brainstorming here, but perhaps it's possible to split the result set up into smaller groups using filters, and then join them all in the end?

[tb01923]

I thought I had a hacked solution, but I seem to have lost it. I think I can handle it in my code. Will keep you posted, but if the result set exceeds 100 items there is a
{
returnval: { token: 'XXX' }
}
that can be used to retrieve the next lump of stuff.

[tb01923]

@reedog117 I posted this on stack a while ago. we really need a method of calling ContinueRetrievePropertiesEx with arguments that includes the token returned along with some other aspect...

[tb01923]

OK this is what I am doing to solve the problem outside of the library (using the library):

const getVirtualMachines = (vc) => {
    return new Task( (reject, resolve) => {
        const rootFolder = vc.serviceContent.rootFolder;
        const vms =  vc.getMORefsInContainerByType( rootFolder, 'VirtualMachine');
        vms.once('result', (initial) =>{
            if(initial.returnval.token == undefined) {
                resolve(initial)
                return
            }else {
                const thisReceiveAll = receiveAll(reject, resolve)
                thisReceiveAll(vc, initial)
            }
        })
        vms.once('error', reject)
    })
}

const receiveAll = (reject, resolve) => (vc, initial) => {
    const executeContinueReceive = function executeContinueReceive(previous) {
        const args = {
            _this: {"attributes":{"type":"PropertyCollector"},"$value":"propertyCollector"},
            token: previous.returnval.token
        }
        vc.vc.runCommand('ContinueRetrievePropertiesEx', args).once('result', function(current){
            const previousObjects = previous.returnval.objects
            const currentObjects = current.returnval.objects
            const allObjects = previousObjects.concat(currentObjects)

            current.returnval.objects = allObjects
            if(current.returnval.token == undefined) {
                resolve(current);
                return
            }
            return executeContinueReceive(current)

        }).once('error', reject);
    }
    executeContinueReceive(initial)
}

Essentially when I retreive the intial result set, and check for a token. If the token is there I enter a recursive "receiveAll" function which calls runCommand('ContinueRetrievePropertiesEx', args) and appends its results. Again checking the token at the end, and either returning the result OR make another recursive call. It works -- BUT I wonder if this should migrate back into the library somehow. @reedog117 let me know if there is anything else I can do.